From acfad81327fb6bcb23536fff498c3a54ba118b4a Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Mon, 26 May 2014 21:47:38 +0100 Subject: [PATCH] added CCLASH_Z7_OBJ=yes option. --- CClash/CClash.csproj | 2 + CClash/CClash.csproj.user | 2 +- CClash/CClashMessage.cs | 1 + CClash/CClashServer.cs | 241 ++++++++++++------ CClash/CClashServerClient.cs | 31 ++- CClash/CacheManifest.cs | 5 + CClash/Compiler.cs | 70 ++++- CClash/CompilerCacheBase.cs | 7 + CClash/DirectCompilerCache.cs | 48 ++-- CClash/DirectCompilerCacheServer.cs | 10 +- CClash/Program.cs | 27 +- CClash/Settings.cs | 12 + .../Express/DVD-5/DiskImages/DISK1/Setup.ini | Bin 5006 -> 5006 bytes README.md | 23 +- cclash.v11.suo | Bin 261120 -> 267264 bytes 15 files changed, 332 insertions(+), 147 deletions(-) diff --git a/CClash/CClash.csproj b/CClash/CClash.csproj index 3613fb2..1aa466f 100644 --- a/CClash/CClash.csproj +++ b/CClash/CClash.csproj @@ -39,6 +39,7 @@ + @@ -68,6 +69,7 @@ + diff --git a/CClash/CClash.csproj.user b/CClash/CClash.csproj.user index 3da5017..cad999d 100644 --- a/CClash/CClash.csproj.user +++ b/CClash/CClash.csproj.user @@ -1,6 +1,6 @@  - --cclash-server + --cclash-server --debug --pdb-to-z7 \ No newline at end of file diff --git a/CClash/CClashMessage.cs b/CClash/CClashMessage.cs index dc130b3..68629d3 100644 --- a/CClash/CClashMessage.cs +++ b/CClash/CClashMessage.cs @@ -60,6 +60,7 @@ public class CClashRequest : CClashMessage public IList argv; public string compiler; public int tag; + public int pid; } [Serializable] diff --git a/CClash/CClashServer.cs b/CClash/CClashServer.cs index 73ccf93..706a35b 100644 --- a/CClash/CClashServer.cs +++ b/CClash/CClashServer.cs @@ -14,7 +14,15 @@ public sealed class CClashServer : IDisposable bool quitnow = false; DirectCompilerCacheServer cache; - int connections = 0; + /// + /// The maximum number of pending requests. + /// + public const int MaxServerThreads = 20; + + public const int QuitAfterIdleMinutes = 10; + + List serverPipes = new List(); + List serverThreads = new List(); string mydocs = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments); @@ -23,108 +31,175 @@ public CClashServer() Directory.SetCurrentDirectory(mydocs); } - DateTime lastYield = DateTime.Now; - void YieldLocks() + int busyThreads = 0; + + public int BusyThreadCount { - if (DateTime.Now.Subtract(lastYield).TotalSeconds > 5) + get { - cache.YieldLocks(); - lastYield = DateTime.Now; + return busyThreads; } } - public void Listen(string cachedir) + void ThreadIsBusy() { - - var mtx = new Mutex(false, "cclash_serv_" + cachedir.ToLower().GetHashCode()); + lock (serverThreads) + { + busyThreads++; + } + } - try { - if (!mtx.WaitOne(500)) { - return; // some other process is holding it - } - } catch (AbandonedMutexException) { - // past server must have died! + void ThreadIsIdle() + { + lock (serverThreads) + { + busyThreads--; + } + } + + DateTime lastRequest = DateTime.Now; + + void ThreadBeforeProcessRequest() + { + lastRequest = DateTime.Now; + if (BusyThreadCount > Environment.ProcessorCount) + { + System.Threading.Thread.Sleep(60/Environment.ProcessorCount); } + } - try { - Logging.Emit("server listening.."); - - using (var nss = new NamedPipeServerStream(MakePipeName(cachedir), PipeDirection.InOut, 1, PipeTransmissionMode.Message, PipeOptions.WriteThrough | PipeOptions.Asynchronous)) { - cache = new DirectCompilerCacheServer(cachedir); - var msgbuf = new List(); - var rxbuf = new byte[256 * 1024]; - DateTime lastConnection = DateTime.Now; - - do { - // don't hog folders - System.IO.Directory.SetCurrentDirectory(mydocs); - Logging.Emit("server waiting.."); - YieldLocks(); - try { - connections++; - - if (!nss.IsConnected) { - var w = nss.BeginWaitForConnection(null, null); - while (!w.AsyncWaitHandle.WaitOne(5000)) { - try { - YieldLocks(); - } catch { } - if (quitnow) { - return; - } - if (DateTime.Now.Subtract(lastConnection).TotalSeconds > 90) - Stop(); - } - nss.EndWaitForConnection(w); - lastConnection = DateTime.Now; + public void ConnectionThreadFn(object con) + { + using (var nss = con as NamedPipeServerStream) + { + try + { + + while (!quitnow) + { + var w = nss.BeginWaitForConnection(null, null); + Logging.Emit("waiting for client.."); + while (!w.AsyncWaitHandle.WaitOne(1000)) + { + if (quitnow) + { + return; } + } + nss.EndWaitForConnection(w); + Logging.Emit("got client"); + if (nss.IsConnected) + { + Logging.Emit("server connected"); + ThreadBeforeProcessRequest(); + ThreadIsBusy(); + ServiceRequest(nss); + ThreadIsIdle(); + } + } + } + catch (IOException ex) + { + Logging.Error("server thread got {0}, {1}", ex.GetType().Name, ex.Message); + Logging.Error(":{0}", ex.ToString()); + } + } + } - Logging.Emit("server connected.."); + public void ServiceRequest(NamedPipeServerStream nss) + { + var msgbuf = new List(8192); + var rxbuf = new byte[256 * 1024]; + int count = 0; + do + { + count = nss.Read(rxbuf, msgbuf.Count, rxbuf.Length); + if (count > 0) + { + msgbuf.AddRange(rxbuf.Take(count)); + } - msgbuf.Clear(); - int count = 0; - do { - count = nss.Read(rxbuf, msgbuf.Count, rxbuf.Length); - if (count > 0) { - msgbuf.AddRange(rxbuf.Take(count)); - } + } while (!nss.IsMessageComplete); - } while (!nss.IsMessageComplete); + Logging.Emit("server read {0} bytes", msgbuf.Count); - Logging.Emit("server read {0} bytes", msgbuf.Count); + // deserialize message from msgbuf + var req = CClashMessage.Deserialize(msgbuf.ToArray()); + cache.Setup(); // needed? + Logging.Emit("processing request"); + var resp = ProcessRequest(req); + Logging.Emit("request complete: supported={0}, exitcode={1}", resp.supported, resp.exitcode); + var tx = resp.Serialize(); + nss.Write(tx, 0, tx.Length); + nss.Flush(); + Logging.Emit("server written {0} bytes", tx.Length); - // deserialize message from msgbuf - var req = CClashMessage.Deserialize(msgbuf.ToArray()); - cache.Setup(); - var resp = ProcessRequest(req); - var tx = resp.Serialize(); - nss.Write(tx, 0, tx.Length); - nss.Flush(); + nss.WaitForPipeDrain(); + nss.Disconnect(); + Logging.Emit("request done"); + } - // don't hog folders - cache.Finished(); + void NewServerThread(string cachedir) + { + var t = new Thread(new ParameterizedThreadStart(ConnectionThreadFn)); + t.IsBackground = true; + serverThreads.Add(t); + var nss = new NamedPipeServerStream(MakePipeName(cachedir), PipeDirection.InOut, MaxServerThreads, PipeTransmissionMode.Message, PipeOptions.WriteThrough | PipeOptions.Asynchronous); + t.Start(nss); + Logging.Emit("server thread started"); + } + + public void Listen(string cachedir) + { + Environment.CurrentDirectory = mydocs; + var mtx = new Mutex(false, "cclash_serv_" + cachedir.ToLower().GetHashCode()); + try + { + if (!mtx.WaitOne(1000)) + { + quitnow = true; + Logging.Error("another server is already running"); + return; // some other process is holding it! + } + } + catch (AbandonedMutexException) + { + Logging.Warning("previous instance did not exit cleanly!"); + } + cache = new DirectCompilerCacheServer(cachedir); + Logging.Emit("starting server threads.."); - nss.WaitForPipeDrain(); - nss.Disconnect(); - Logging.Emit("server disconnected.."); - } catch (IOException) { - Logging.Warning("error on client pipe"); - nss.Disconnect(); + while (serverThreads.Count < MaxServerThreads) + { + NewServerThread(cachedir); + } - } catch (Exception e) { - Logging.Error("server exception {0}", e); - Stop(); - } - } while (!quitnow); - Logging.Emit("server quitting"); + // maintain the threadpool + while (!quitnow) + { + foreach (var t in serverThreads.ToArray()) + { + if (t.Join(1000)) + { + serverThreads.Remove(t); + NewServerThread(cachedir); + } + if (DateTime.Now.Subtract(lastRequest).TotalMinutes > QuitAfterIdleMinutes) + { + quitnow = true; + } } - } catch (IOException ex) { - Logging.Emit("{0}", ex); - return; - } finally { - mtx.ReleaseMutex(); } + foreach (var t in serverThreads) + { + t.Join(2000); + } + + + Logging.Emit("server quitting"); + mtx.ReleaseMutex(); + } public static string MakePipeName(string cachedir) @@ -151,7 +226,7 @@ public CClashResponse ProcessRequest(CClashRequest req) break; case Command.Run: - cache.SetCompiler(req.compiler, req.workdir, new Dictionary( req.envs )); + cache.SetCompilerEx(req.pid, req.compiler, req.workdir, new Dictionary( req.envs )); rv.exitcode = cache.CompileOrCache(req.argv); System.IO.Directory.SetCurrentDirectory(mydocs); rv.supported = true; diff --git a/CClash/CClashServerClient.cs b/CClash/CClashServerClient.cs index 0b61ae0..8e40c1d 100644 --- a/CClash/CClashServerClient.cs +++ b/CClash/CClashServerClient.cs @@ -33,19 +33,23 @@ void Connect() if (ncs == null) Open(); - for (int i = 0; i < 2; i++) + try { - try { - if (!ncs.IsConnected) - ncs.Connect(100); - ncs.ReadMode = PipeTransmissionMode.Message; - return; - } catch (IOException ex) { - Logging.Emit("error connecting {0}", ex.Message); - try { ncs.Dispose(); Open(); } catch { } - } catch (TimeoutException) { - } + if (!ncs.IsConnected) + ncs.Connect(100); + ncs.ReadMode = PipeTransmissionMode.Message; + return; + } + catch (IOException ex) + { + Logging.Emit("error connecting {0}", ex.Message); + try { ncs.Dispose(); Open(); } + catch { } } + catch (TimeoutException) + { + } + // start the server, but lets not try to use it here, the next instance can try { @@ -59,11 +63,10 @@ void Connect() p.StartInfo.WorkingDirectory = Environment.CurrentDirectory; p.StartInfo.WindowStyle = ProcessWindowStyle.Hidden; p.Start(); - + throw new CClashWarningException("starting new server"); } catch (Exception e) { Logging.Emit("error starting cclash server process", e.Message); } - throw new CClashWarningException("failed to connect to server"); } public ICacheInfo Stats @@ -149,7 +152,7 @@ public CClashResponse Transact(CClashRequest req) { Connect(); CClashResponse resp = null; - + req.pid = System.Diagnostics.Process.GetCurrentProcess().Id; var txbuf = req.Serialize(); ncs.Write(txbuf, 0, txbuf.Length); diff --git a/CClash/CacheManifest.cs b/CClash/CacheManifest.cs index 51c7185..1c2861e 100644 --- a/CClash/CacheManifest.cs +++ b/CClash/CacheManifest.cs @@ -35,6 +35,11 @@ public static CacheManifest Deserialize(Stream stream) /// public string CommonHash { get; set; } + /// + /// Hash of the pre-existing PDB file before this object was created. + /// + public string EarlierPdbHash { get; set; } + /// /// non-null if this entry was made by preprocessing the source /// diff --git a/CClash/Compiler.cs b/CClash/Compiler.cs index a9d1c1d..68750aa 100644 --- a/CClash/Compiler.cs +++ b/CClash/Compiler.cs @@ -14,6 +14,7 @@ namespace CClash public sealed class Compiler { static Regex findLineInclude = new Regex("#line\\s+\\d+\\s+\"([^\"]+)\""); + public const string InternalResponseFileSuffix = "cclash"; [DllImport("kernel32.dll", CharSet = CharSet.Auto)] static unsafe extern IntPtr GetEnvironmentStringsA(); @@ -170,6 +171,11 @@ public string CompilerExe /// public string[] CommandLine { get; set; } + /// + /// The arguments that should be sent to the compiler by us or our caller. + /// + public string[] CompileArgs { get; set; } + /// /// The first source file. /// @@ -211,6 +217,8 @@ public string[] SourceFiles } } + public int ParentPid { get; set; } + public string ObjectTarget { get; set; } public string PdbFile { get; set; } @@ -218,12 +226,16 @@ public string[] SourceFiles public bool PrecompiledHeaders { get; set; } public bool GeneratePdb { get; set; } public bool AttemptPdb { get; set; } + public bool PdbExistsAlready { get; set; } public string ResponseFile { get; set; } + + List srcs = new List(); List incs = new List(); List cliincs = new List(); + public List CliIncludePaths { get @@ -244,7 +256,8 @@ bool IsSupported { get { - return (!Linking && + return ( + !Linking && !PrecompiledHeaders && SingleSource && ((!GeneratePdb) || AttemptPdb ) && @@ -378,6 +391,7 @@ public bool ProcessArguments(string[] args) var opt = getOption(args[i]); var full = getFullOption(args[i]); + #region switch process each argument type switch (opt) { case "/o": @@ -423,6 +437,12 @@ public bool ProcessArguments(string[] args) case "/Fd": GeneratePdb = true; PdbFile = Path.Combine(WorkingDirectory, full.Substring(3)); + // openssl gives us a posix path here.. + PdbFile = PdbFile.Replace('/', '\\'); + if (!PdbFile.ToLower().EndsWith(".pdb")) + { + PdbFile = PdbFile + ".pdb"; + } break; case "/Fo": @@ -448,7 +468,7 @@ public bool ProcessArguments(string[] args) break; default: - + #region positional or other flag options if (full.StartsWith("/E")) { return NotSupported("/E"); @@ -462,7 +482,15 @@ public bool ProcessArguments(string[] args) if (opt.StartsWith("@")) { + #region response file ResponseFile = full.Substring(1); + + if (ResponseFile.EndsWith(InternalResponseFileSuffix)) + { + Logging.Emit("cclash misshelper internal response file"); + return false; + } + if (!Path.IsPathRooted(ResponseFile)) ResponseFile = Path.Combine(WorkingDirectory, ResponseFile); var rsptxt = File.ReadAllText(ResponseFile); @@ -486,6 +514,7 @@ public bool ProcessArguments(string[] args) } return NotSupported("response file error"); + #endregion } if (!full.StartsWith("/")) @@ -509,23 +538,28 @@ public bool ProcessArguments(string[] args) if (d == "..") d = Path.GetDirectoryName(WorkingDirectory); - if (!Path.IsPathRooted(d)) { + if (!Path.IsPathRooted(d)) + { d = Path.Combine(WorkingDirectory, d); } if (Directory.Exists(d)) { Logging.Emit("cli include '{0}' => {1}", full, d); - + cliincs.Add(d); continue; } } +#endregion break; } + #endregion + } + if (SingleSource) { if (ObjectTarget == null) @@ -542,12 +576,36 @@ public bool ProcessArguments(string[] args) } if (GeneratePdb) + { + if (Settings.ConvertObjPdbToZ7) + { + // append /Z7 to the arg list + var newargs = new List(args); + newargs.Add("/Z7"); + AttemptPdb = false; + PdbFile = null; + GeneratePdb = false; + PdbExistsAlready = false; + args = newargs.ToArray(); + } + } + + if (GeneratePdb) { if (Settings.AttemptPDBCaching) { if (PdbFile != null) { AttemptPdb = true; + if (PdbFile.EndsWith("\\")) + { + AttemptPdb = false; + return NotSupported("Implicit PDB folder+file requested"); + } + if (FileUtils.Exists(PdbFile)) + { + PdbExistsAlready = true; + } } else { @@ -559,16 +617,14 @@ public bool ProcessArguments(string[] args) return NotSupported("PDB file requested"); } } - } - } catch (Exception e) { Console.Error.WriteLine(e); return NotSupported("option parser exception '{0}'", e); } - + CompileArgs = args.ToArray(); return IsSupported; } diff --git a/CClash/CompilerCacheBase.cs b/CClash/CompilerCacheBase.cs index ccefc3b..78bc690 100644 --- a/CClash/CompilerCacheBase.cs +++ b/CClash/CompilerCacheBase.cs @@ -71,6 +71,12 @@ public CompilerCacheBase(string cacheFolder) public abstract void Setup(); public abstract void Finished(); + public void SetCompilerEx(int parentpid, string compiler, string workdir, Dictionary envs) + { + SetCompiler(compiler, workdir, envs); + comp.ParentPid = parentpid; + } + public void SetCompiler(string compiler, string workdir, Dictionary envs) { if (string.IsNullOrEmpty(compiler)) throw new ArgumentNullException("compiler"); @@ -268,6 +274,7 @@ public virtual int CompileOrCache(IEnumerable args) { if (IsSupported(args)) { + args = comp.CompileArgs; var hc = DeriveHashKey(args); if (hc.Result == DataHashResult.Ok) { diff --git a/CClash/DirectCompilerCache.cs b/CClash/DirectCompilerCache.cs index f3a50c7..a97d5da 100644 --- a/CClash/DirectCompilerCache.cs +++ b/CClash/DirectCompilerCache.cs @@ -78,37 +78,28 @@ protected override bool CheckCache( IEnumerable args, DataHash commonkey } } - foreach (var f in new string[] { F_Manifest, F_Object, F_Stderr, F_Stdout }) + if (comp.AttemptPdb) { - if (!FileUtils.Exists(outputCache.MakePath(commonkey.Hash, f))) + if (comp.PdbExistsAlready) { - outputCache.Remove(commonkey.Hash); - Logging.Miss(DataHashResult.CacheCorrupt, commonkey.Hash, comp.SingleSourceFile, ""); - return false; + var pdbhash = hasher.DigestBinaryFile(comp.PdbFile); + if (pdbhash.Hash != manifest.EarlierPdbHash) + { + outputCache.Remove(commonkey.Hash); + Logging.Miss(DataHashResult.FileChanged, commonkey.Hash, comp.PdbFile, ""); + return false; + } } } - if (comp.AttemptPdb && comp.PdbFile != null) + foreach (var f in new string[] { F_Manifest, F_Object, F_Stderr, F_Stdout }) { - var cachedpdb = outputCache.MakePath(commonkey.Hash, F_Pdb); - if (!FileUtils.Exists(cachedpdb)) + if (!FileUtils.Exists(outputCache.MakePath(commonkey.Hash, f))) { - Logging.Miss(DataHashResult.CacheCorrupt, commonkey.Hash, comp.PdbFile, ""); + outputCache.Remove(commonkey.Hash); + Logging.Miss(DataHashResult.CacheCorrupt, commonkey.Hash, comp.SingleSourceFile, ""); return false; } - - // check if the target pdb exists already and it's contents differs from our cached copy. - // If so, fail the cache hit as pdbs get merged not overwritten :( - if (FileUtils.Exists(comp.PdbFile)) - { - - var pdbhash = hasher.DigestBinaryFile(comp.PdbFile); - if (pdbhash.Hash != manifest.PdbHash) - { - Logging.Miss(DataHashResult.FileChanged, comp.WorkingDirectory, comp.PdbFile, ""); - return false; - } - } } return true; // cache hit, all includes match and no new files added @@ -193,6 +184,8 @@ protected override int OnCacheMissLocked(DataHash hc, IEnumerable args, return rv; } + static object RelatedMissLock = new object(); + protected virtual void DoCacheMiss( Compiler c, DataHash hc, IEnumerable args, CacheManifest m, List ifiles) { @@ -214,6 +207,17 @@ protected virtual void DoCacheMiss( Compiler c, DataHash hc, IEnumerable m.IncludeFiles = new Dictionary(); m.TimeStamp = DateTime.Now.ToString("s"); m.CommonHash = hc.Hash; + if (c.AttemptPdb) + { + if (c.PdbExistsAlready) + { + var ph = hasher.DigestBinaryFile(c.PdbFile); + if (ph.Result == DataHashResult.Ok) + { + m.EarlierPdbHash = ph.Hash; + } + } + } #endregion diff --git a/CClash/DirectCompilerCacheServer.cs b/CClash/DirectCompilerCacheServer.cs index 5792cae..7df6c5a 100644 --- a/CClash/DirectCompilerCacheServer.cs +++ b/CClash/DirectCompilerCacheServer.cs @@ -154,9 +154,13 @@ public override Dictionary GetHashes(IEnumerable fname { foreach (var filename in tmp.Keys) { - hashcache[filename.ToLower()] = tmp[filename]; - rv[filename.ToLower()] = tmp[filename]; - WatchFile(filename.ToLower()); + var flow = filename.ToLower(); + hashcache[flow] = tmp[filename]; + rv[flow] = tmp[filename]; + if (!flow.StartsWith( Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(comp.CompilerExe))))) + { + WatchFile(flow); + } } } } diff --git a/CClash/Program.cs b/CClash/Program.cs index db78120..3860a87 100644 --- a/CClash/Program.cs +++ b/CClash/Program.cs @@ -41,6 +41,14 @@ public static int Main(string[] args) if (args.Contains("--cclash-server")) { var server = new CClashServer(); + if (args.Contains("--attempt-pdb")) + { + Environment.SetEnvironmentVariable("CCLASH_ATTEMPT_PDB_CACHE", "yes"); + } + if (args.Contains("--pdb-to-z7")) + { + Environment.SetEnvironmentVariable("CCLASH_Z7_OBJ", "yes"); + } if (args.Contains("--debug")) { if (Settings.DebugFile == null) { @@ -63,17 +71,28 @@ public static int Main(string[] args) if (Settings.ServiceMode) { for (int i = 0; i < 3; i++ ) { - try { + try + { var cc = new CClashServerClient(Settings.CacheDirectory); - if (args.Contains("--stop")) { + if (args.Contains("--stop")) + { cc.Transact(new CClashRequest() { cmd = Command.Quit }); - } else { + } + else + { Console.Out.WriteLine(cc.GetStats(compiler)); return 0; } - } catch (CClashWarningException) { + } + catch (CClashWarningException) + { System.Threading.Thread.Sleep(2000); } + catch (InvalidOperationException) + { + Logging.Error("server not running"); + return -1; + } } } else diff --git a/CClash/Settings.cs b/CClash/Settings.cs index 59f7484..7782da6 100644 --- a/CClash/Settings.cs +++ b/CClash/Settings.cs @@ -57,6 +57,18 @@ public static bool AttemptPDBCaching } } + /// + /// When an object compilation with pdb generation (Zi) is requested. Instead + /// generate embedded debug info (Z7). + /// + public static bool ConvertObjPdbToZ7 + { + get + { + return Environment.GetEnvironmentVariable("CCLASH_Z7_OBJ") == "yes"; + } + } + static bool EnabledByConditions() { return ConditionVarsAreTrue("CCLASH_ENABLE_WHEN"); diff --git a/Installer/Installer/Express/DVD-5/DiskImages/DISK1/Setup.ini b/Installer/Installer/Express/DVD-5/DiskImages/DISK1/Setup.ini index a3d56276b5869c1ff4ab2cfdcc8e41670083199c..add1b0bb22292dcf766b7c56548cc5e307c6eef5 100644 GIT binary patch delta 84 zcmeBE?^EBP#q4gxU<`z&3}y@l46Y1@47v=KK(-~AWddZGGFSk`EPyO01`{C5guxuB Y&WOPcsKy*fy8=}hGB|B6WR?{G00`>~ivR!s delta 84 zcmeBE?^EBP#q4g*V9MYG#O4fU48}mL%U}S8ra+bngAs!Tkkn;x1gbCtit931g4H=O SI0I!2fjn1;n$3mGvH}1HBMWZ; diff --git a/README.md b/README.md index c37576b..e408a7e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ CClash is a (.net based) compiler cache for the Microsoft 'cl' compiler. -It is aimed at fairly simple use ( eg, `cl /c file.c /Fofile.o` ) and will cache object and pdb files rather like ccache does. +It is aimed at fairly simple use ( eg, `cl /c file.c /Fofile.o` ) and will cache object files rather like ccache does. Due to the nature of PDB files on windows cclash can't easily cache these so you might be out of luck. However you might want to experiment with the CCLASH_Z7_OBJ=yes environment setting to see if it gives you what you want. CClash has been inspired by clcache (https://github.com/frerich/clcache) and of course ccache (http://ccache.samba.org). @@ -15,24 +15,21 @@ On subsequent builds, the key will match, giving us a the manifest file, we chec Those who know ccache will recognise this as quite similar to the ccache 'direct mode'. ( I read the ccache man page before writing this - http://ccache.samba.org/manual.html#_the_direct_mode ). CClash has a preprocessor mode but it does not work very well and is best avoided at the moment. -CClash has a couple of extra twists to get a little more speed, if you set the `CCLASH_SERVER` environment variable, a short lived ( 5 mins max ) server will be spawned. This server will take over the running of the compiler and the hashing of input data, so common files will only be hashed once! The stats below are from using the server mode. -Also, you can set `CCLASH_HARDLINK` and cclash will attempt to make NTFS hardlinks rather than copy files which should also be faster. Don't try this if your files are on a different drive or ntfs volume to `CCLASH_DIR`. If your system has a significant disk I/O overhead this might help you. - ## How well does it work? -On my (not very good) windows 8 machine, I have a simple test by building openssl under nmake. These tests were all done _after_ running the Configure step. +On my (not very good) windows 8.1 machine, I have a simple test by building openssl under nmake. These tests were all done _after_ running the Configure and do_ms steps. OpenSSL windows build | Duration | Time Difference | ----------------------------------------------------------|----------|-----------------| -average build time without cclash | 288s | | -cclash enabled first run | 464s | +61% | -cclash enabled second run | **128s** | **-56% ** | -cclash enabled first run (CCLASH_SERVER) | 391s | +35% | -cclash enabled second run (CCLASH_SERVER) | **108s** | **-63%** | +average build time without cclash | 405s | | +cclash enabled first run (CCLASH_SERVER) | 428s | +6% | +cclash enabled second run (CCLASH_SERVER) | **123s** | **-71%** | + +The first build with a clean cache costs about an extra 5-15% in build time for each file. Subsequent builds (with no source changes) give about a 50-80% reduction in build time for some operations. -The first build with a clean cache costs about an extra 30-60% in build time for each file. Subsequent builds (with no source changes) give about a 50-65% reduction in build time. +This build was done with CCLASH_Z7_OBJ=yes to prevent openssl trying to make pdb files for each .obj it creates (you still get the desired pdb files after the linker calls) -So.. For cclash to make a difference to you, you would want to be in a situation where you will compile most of your files more than 2 or 3 times. In my case, I wanted cclash for a continuous integration build so it _should_ pay off quite quickly day or so +So.. For cclash to make a difference to you, you would want to be in a situation where you will compile most of your files more than 2 or 3 times. In my case, I wanted cclash for a continuous integration build so it _should_ pay off quite quickly day or so. Compared to good old ccache this isn't too bad (ccache with gcc costs about 25% on a clean cache). Your milage may of course vary greatly! On a multi-core machine with GNU Make things go quite well too. @@ -40,7 +37,7 @@ Compared to good old ccache this isn't too bad (ccache with gcc costs about 25% There are two ways. The first is easier if you can adjust your environment. The latter might be your only choice if you wish to use the Visual Studio IDE. - * Add the folder that contains cclash's cl.exe to %PATH% before the visual studio compiler + * Add the folder that contains cclash's cl.exe to %PATH% before the visual studio compiler. or diff --git a/cclash.v11.suo b/cclash.v11.suo index 35b7139bf6a243ed730d454c48aa08d79bbc721c..9d0f7eccdaf0780a36cece1b51c5c7e0a21bd71f 100644 GIT binary patch delta 14485 zcmdUW3tUxI_WwD1Ul5S+f{2KQSG+1dz{~4`ig~pYv)5W$~yCzZy#aeLcqs0HfDi!Nt1(14&=`bD55{JFgce0jXi?n9;C{On6Hpgg6n(DiZ=m4P zqN`2ZI|&i=LS?SjakrKKj8Q^mqL1y-MEjm_GV&?z8R!@+7@8uqn+8aCqV~ppGQeeB zp(v*SCIG~e-3Q172(m)#@qn=aA+8IS$*kCIWsk*ZUIJJOumQvnI>BIfw}m4o&eh%ML?7m!UC z=z8A1iiXHb*jurq?Al~csR&fc01A5)_{E?uF1uFGHg3^*SWp(LtRQ@W&Zk5#E`gxm z(pxa;b^+}h*n*J3avbYJdNNbDXtyh9dxCwtU?KUd%;whKbl0LGZY`@_xWXQo>?RAV zWbdaAVb(xMI0v24-#XcWBSh^Rm5upK4C~>xXC(z!dvteLRb1 zT9)#N*26`I?G$leND6#cwh_~EWOw+Wf zD}7!qzLsokSUy&_EpAK^26W?difK28=^VvuF>|zQD>QOX)zOu*w%=;VlXC0&mIB!x zeU+R;5Qx6%z8Q2UZecg-LYaPfFP5NN&XT)ExCtNR&A!#au7&d!J6%~YYgB{1DTSqU zHMClr8f39cg!31D`zk9LEwQ<)4tayUF%aNS7`4bwI{r=Iz%@#A%199iN#_r$}Xu$n9C9$$PF!#s_~aD znWa*x0>4#KxKxOiGaLT2>x-fSw=3RVeCBVta#TpgpUS8Gm@Q=nJ6Yf!S_o`0cCuuZ zN-(C8oqO({%UNk+wg`_L=qC`qV*9 zv)Aot-?Y$Mn)cZ%GZfigsG$Dbv>L3Ku25U!O4`P3H8t9^uiuOr5MUhGc^KQ!BQWiG z1$Ww`bbz*7pN(k+yZ!f`f988KJm6DbzUEO{Asd*y;ug16!n*313|G3YNZImFL0(qs z6DV(I%hwGKe-YJffR_Mbv+uxtC!iIu3!sOd(5zhjBzAIL|8DWOHN69r?*N~()A#gV zTT$vJ+^@-*t_&20BSwCM#%}>P0RIGh4^Z3>?zaE~+2X(fx36XAHE68?;9|GNr7Bxf z1a@V+b|Gx;_(L8dbrdm1w82@_OhBU3HA3}x-=vQ9=!8DOfzt28YSPp|DAGS=3tmmL zxl_y`9RBD`jk{btveF`3SC_~SJGL%T zti0BX)ppI)i021&mDXzImsP9V?zS5g4V!^I8_);^7{Io?bGWy){_V7sb|wt$DJ1 zb8UWC`_uj~^lnf0bzco;)<`Vj>F|M1qxcM9JzxW%3GghS8SornBj9?_!S~{!BGJ_p=rO?goYAV=AMD{P z9tE=C+RNeE6{pSQgWX!^J65_QqkPuk=sHfymn?vp=q>|TBrE&swSJDae)8Ys)@>gL zxT%PVcR_^PFM|6obv|4yKp?OxWWuGpV7K=9I+pT9AIl!C7f*HV$zFPIbnBC!HOXw- zcj*p8quku9#akwuj_j08Yx51vVE0zHoXPGV7y#V{0>r}fblyd|wbZ^&?od$3oeJ9f zS%PLKo=b02kmaGewkc?Wg0{)sMQ|C1DE5l0I35}&2JZm&fhb^;D6)j>52JcsPz@PG zvshM&4-1|iiZSDO;x$aTA`YylT*HKMBS~I|Bbwo3FfUZmE=0=YwUaPz*H)BzO#*)tvq_^m z_3&&L6UPP0#1BaNCc`vBSkmbbcJf_+9z2D(^SU~WTr-(ydEHQyA}51}CZ2-YRF~5B z+z%elOd*+@3=a?un|NBs$KFr$ylx6u)K7tc_(|ter@Q9+W*SOhE`c}KlXS=Y2gx3p zH%t>lh6FSn`9X*jpd_^)GwwK9v!jEhxL-^hqP~5s< z;3;PYxJ$|175D*OUrh|E9jus#DHSK6p9k~ZHy3g!Lz7lO^{19&lwtx+ zCYW^na@P(FT`r`oBhB1WCkT8Y>|ka9uU|~H@<`!d#_|0RV?~P|795BTm) zx9t)5mja%6jAZciM_@||(z$w2Q#NXSJvagDnLM-}jGb>l6)Rc5>l;WfVJ(^5uK`R5 z^XFMHAiMoh;29nT0cG1BSt&>zCS$pErC?T&(yj!G`(}_*c)0i_e9H@)h@YIzzd8-g zWo;rp>YhE{0wje_LksA@b>;HNt>ge73t!LucaX8ZF;b!utxKe0{7F)d>Q?<Qo|(M|3y$R(%fE2|Pg%|W?_zF@$mETG1nuZQqN;4-WA8w>$~gASTz_8w z4jGSg2QaT#(b;!F05+b<>)&-P&d!4_GsOwdhkQ=YUCY> zXJwO#jqk$~HMT=R1rT`x>oUiWr?-RkgMzl=#IugL%;(bkq9iu@s`t^Wl!A|<1P^_X z$9;ej_Gm6UY1Xqde+_U1bfaJNbQ2zWKd+2}jb(lWHlxphuXD#xQZ{4#F&Ftkq6F`8 zj3<1ElCm#O9fbmHM^Sa|elI@u6Bs8}Accn%kx)1UrOBVg|H;oWHG*#L)Uyo7B56_# zvYsSZMg@GAdh8)dr&T^+U(Q6q>tpCg(b)?hp|oo|csp?%(%e4^9d+35M7T+ zbPfcuJy-F)|3HXS=G@*NyDjJ+VD!w#m_2VL)q}~Me8=a6sUj;F&7X^GK;&GzFM!;- zomh_S3m9zyq4g2iiRg8qV znY>h#z*?@{s3x-ZDGNQAA;kwwcyjp=oLbA*E?ZlpfpCgy4WguYq;^42u}N)T$gt~kC`oIQ<9XAg z2&pgRk`UQTO_j3U(69K?%g)j1T(8H$o8mO~i_!YNbh$ZXben#l^zAD6RzAkWuFCxe#QR0rMy; zGsQrWpASV7;Nyn-BoYyhuVq*(a=WJG}oK58@_%pcE!3CtZ$D`*!<;^Ris z?%bXM`$JPE9U+b4m80o>j<|HVHb=z{s_n$>3u$0uKJ^#C3`FB9yYAethz{YVd}``^ z2FF(7ZJwXBiSL*}t9afd>gHapp6?IM#q@En(CrR7nd)SI?{pd^@8Zi|qjk5V$j^(R zm0o2f)z!1ir4@KHpp2X{g;p!{I+dBLtMg}?JDRu3%50V16;wODK0WBF(x)LEIKVUEo@j!a%xyiWMpJ?baaw2 z$r!zO5#KSHKF?z&(l_NT9o!A$PfVh(%C9R8IiQsk8Ec3$#3qNO#3dWUqKBm>g~g>N zrQ#M9B7Ev4)VKBsnMHd%DhULdb#DqoVN1LOIBV*0R2m>Bl z#p$4Om6-m;qRD=|b{dV~+h^Qm1rh^r%ILf_ZQ=!(coCLzcT$PI?JiuYf7>aPGx)Mx zy1$#LqdBxqye1z2ND<1Y5@wR&d!VCV)AkX{gZQG?=mFoOw>5u?@^t>f47yLeeD1i! z0eoLT${m)L7_hek%e(FODEAb2cQwDnfIc0#>S?X0s#4x*cZomZK)i`lbn-LCm=s{| zps#6r66JJ3|E?CA7%;vAH?&QB$|AI$a=X1`O{W0?F&$X`ZQ_huY|NB9jXQCvP`2wb zISVPjmc^^OaZRt>m4$zv>mGo=uur8Q`U)A!T>({0s0EhdX4tvPtBnIKH}?) zAIy-;IkX%9I)^^eSV?WW3urUpDHa;oIUH!?l}}Pa9n0J3LW+b92=7o6ZA_uB%eAV9 z_4ZDN$1KH>a9J?&g3QTe6ll1(| zejBUor+>RaFCU(}`!T$!8^r34`|&+h^o8AhWLf6gVbtBxDVn?@^UJdk8bQUKud1T{ z4*Ma5965w%%%{(@%;Cny1$3(H#^pL_KI<4QVLvlwFgX=H?{dTQEa8cQ|4p5i@^ulPY$DcwUmbKgbOV9(Jw+#2vk zv$Ir>=45HOlp}H5W7O2rKnZU{LB7P^>NRTjMSMKKO`QvC`erhz9CU_4`;79UkIQLJG$1lx* zm89acwJb>oZgkQ-$X^9HO8_&SZDDxCiv$BP3dRKVn9)-XJRRPNh`U-ZM%@Z6idqF) z%R96;0JfxRunaHkRBz!c!3z}sDuM4#zciR%oe1}T0a=15J+Gt=YNWqQUV)Ti+g*|g z{%d6oKO3``{tw3NU*e}z{}ZW+uaX4(iv>vRc1K{x?zugJ8$0~7rY#-i9%=;N12py0 zw7nq;B7)DmBhO3h)-hbVEz3~TB?O8PuVjo>V&hdqr;Qtlr2+nU^mopvVtESRV{%vfRz4=>tcu0A0 z8uhq)l63b}GipwS5)G91!5zL0?}p($h3C}42*$&&W_Nf(^$Q{qYs4X$>L>GouT#8d zC8BVN(~r75Xa@!M?B1?r;lR}4lTk53yp2`IiI`O8d@o%D%0*}w<5kKoO2R*jm@iTe z;hP1Ag=nb(zNowKel%mJaC}{SkK`rJD}P~>h((l$az=>^%pqQP=VP8y#F$x%^}nb5xDqV#&JH8J1M$1=B*+;v~8R1H~XUgbPu`t9l&*w+QX| zm_`-ZC1SR-Ke=N%HH>8-lIj#(t3pi;e(F+MrEY3W8?8(%9*TujI(F+|_r5wjAd5Af zhvDkFi0=d{Kx6@IK+zySZKl25WMQEEZ8IIg+wy3(JeH5Eq8aVAjWOYmB|Z5A!!xwCMnb9E-2Ejn_=PY9nr6GsET;8Q*4w^ztyL;3Y$n#A`N z26_)XOtblod33yg2-fD{zV#_b43Q@Ll#7A? zNZdV`Z=Huzr_y;eK~7`KKIzV_^JyKQlZMl0Z86dQGzoSf310I*ygAEu_C z`S|n@?an^%)pNrVnufd?-@?q}IaS2t*UgUk&3Y^mPIvry<|BBLdZ~f>Iz)afs~N8P zeeCX|_@>Wc`;wZ7QoKvvy!J~vbaw--kvor5PiDjin<8hn{p@Fl$e`N@T_+dv9gIxl zp;cr!53tf*zx?y@EennKT(vvu?Tt%n5J3lLpSWSD-=$IHjZZ+-G z#Zp}zo>84#jt@x-t>z*={UVLk#}pf5qRa)cVFiU|b6B(?zaT6wI@TB#6&F!ZSezd> zEH2vEc!BCQkX&_1py#h0wywg`$#;d%E{9e-&Rb$4|h zxIlvvs;YBJECo3!q{8(;S02jP{jgXbe^90k*4*} znQ~+mV*y&azbI7ZE- zvP^I+w{#UXx}^I5qvbKK>4j4HKkMl*#fkS-6x0d@xn$&T8i}5VMBuF6!-%(Lc{;gC zSs{E|L`XpT5oHd$=37hVn$sp& z%)0U_U0Hb*?!`LmEVB;BrL(c622|?`%gZXtEoO^V6fM^L5{oW;d}_A9z+tYn#9WO5 z`Bq((xw57N1jJ|sAq~f^=6U5+bHa7H4C_c;Ni}dlt;}pGG8d_X1ag(R)SO>!)|FTa zOKXbmO2CWdkLb--KaBT;KVEYv*xQ?6tB8NX5&z!7yuAX|f3gvaz|Y3ubB7K^P5%!W zhETVAyrrhJG(i**lGH7?^e$U2HZlrZF2ho7%`YvTG^@m1S_D1T;MY=a)fHBm^R24L z)#b%jY*px?2+A%gE-A#~lv{Mx{F%A}b76iBR3y}uje4>eh84?5(N&wRReu&wd4!(+`ESx49~#ME z`NzAd^}gXlWyJvglY6MIqwO1n8%JUe^^^JF)iju2`keZT10=4wMB|lx#FkBsZ}}Pl LsqH#7{qXZ&r?~hb delta 7332 zcmd5>3slv`*8k7UpThwUhr>feK;$K1B7%@2ARZA&5dmLaO)r8)il8A6Q&2p%w9m{8 z<PR$gt4LTgWl~_RxFG+vDCIK5})v=JNA(yH;joWaw4+_CDv}wU*Xut=n4P;P332 zJu`du?3ul1=C5Uk$MKXz15$ zeDa}#Y&!BSav87V9$a4L^os$43TdwswZ#UE=ajh0OHTX`Mjs#qfVIY{-0$%s2# zJOk1-2v39$!WS_Wfz5jcAT=WTy2`|(GzQBP5vOr94_93u7ai=Pc}_aG=Lr|X-9^8^ zPEIjBJ;RV^cJbxgYW(1SSeb>4G#7Ia(hOHV)(|y7Z;42frxwr%~OXo01=7E zMNmt!2Gt#9@}6O}@{mWEsoqt!5~&4Y)h+b0&a{waS76cpsI29+Jeo|~-T(Ye7A1L< zqOsNWjl~03k`bM(o*ptH%L`;7&^TvuzeYf5umBw^HdUyug!aPqg_Gj9p8!L{v*|FXGSO{hx$N{enQ>mRS z@QN=UijvP@U*`6QQpZ;HHy6aotdRxu_3n`i{wi?wxj{%F|R0R7fV2t zjCv|cB`eCYWidLz73Bd28~2J~uywBpmWO9KGGSB-lSRVk)*Knny{sXfvU6v?95*0= z1o!dDT?wMt(h7F7wIp zVC15-Qyfj^V&~H%PS*4ROdPR zjf}N*CL36Dl-l@}NSa~9Mes(L5IzWBgdf5m(H9Yb=!XbI^hX3Cf)OEz0fx*IXG>ye^%a&Nl^r-w_VA0XV&yi9)+UuEWM87j-a0R zV2|FO;L|Alw0#D3Ap^@YVQDVl_|JOJ0q1f)fK6$Ywsqda498Hj8rP!Sx&=8?Z;R#; z+`1uCrLgMQjghiuL%yC)5IQYLsCBj~yEZJ>1z@k_23h+=j!K)7TDwu0ebRPi)@0NbJy|`cKxWQ~*Oj1vrxn}PKj^r%VZ2IvfZDMbNAZTNjDy&VK?|Zh}0} zf3&rsn&ruwh8`9qYc@V+4eMlI{>0Y}t5968ugkHEMoRO&I@@T=0oJxWm6@7~?|V7; z{F=-O5u#)>PkPj$$ea~c??X{deNOIMcksy0kA}RoWoBDbC-Zt6hc?nq!jvo7BR#PA znivFwz70Opw>__Yq|olJe&wh$=I$)e7a)@^bX377GI@Dty^bzcCQn&A;1TPqsvK2j08<`| zRZXZSnDdx_ASoA!r66n=OcX)Vwemc+iCNd)AF>%a&mguSo<%evo0Bq z=+u1`nXe&Q5xWrK$g?5cjd&f=j?fT$m^`^Qy6-!fJ%IQN;$6hQ5m%hY`&x#VgvgMq z$e~fLA$<>l*&E5axmuTj1N2jEwVP`>`|D3O&$iBIxq;Rtg^q4a);51;l6~7Ebq!eR z&t!UAT==t?C4o&ypF?a#JdfChAUbI*RlgV?okefD0n&D>~VuwomKwf|(TiDi9 zOsa9F5H}P;@r=${3S=}9+JcUSj*Uw;5QE@Y%ZlbrCMwW9tasl?|7|;5+obK~O!l*d zpRiuuRc6{)p~`r@UY;*CY^?B?vs*DcIuNt0*?FT<171LpD7oiFq%DZ;h#iWpDQzxm z-oXtFHRlLHGcdGl7Bg64lnK5--8cbd;%A}`e^!R%F_M%No z4zjVmtU!I}fE^ANTwTPXWv-@&#w4~x9@3&kPmiD6_-Qb-dNTuDdY+wxw!|6kz$h)sz)mo&t)5-h!$%)8y_UC?agV@i zyyo7gR4FBI4EopcncXiNfBf$4x(i{U8-c3l@?aOwxKwY^F(rcHi{_4NC1n57Sz z1=s39Wv;S7es>^PTd<#f;Q?(+S?KMIEp57`a|SEsWLzlYI^RzRgcOVQSS z%-$OW86GTJ>w1eF^HH#$iet579!5 z{G8B^ywAGb93#S3Z$3kN^G@EvSP>1oSPz}6d5*T;hga*Np@kbYV?4jaT9UX5_Hv#L zx#c`So1Ma6V9@q04}p#@9DUB1gM8TD0n*utDgvl2Agtau-TVKuJTDB+Mr&BOE!R$(T!a@J3-S0XbQS zk(f)sb0R?*j(;Pdc&vzoQ>j7+O_?GM4nHa4`()$EM~9zYc?o=S#bVg{j2HzEZWe!r zx1@LxECu3CDB2`i;f*K7tHi4W`|~0Z_U4F}z40)hgTawX%{NKF3v=d8MJSpk0s<0U zEs4+xmVn6I3#$smI=H%7Bsp67(KiKWQti|OADXahibw#T@u&|8@;Qv}qTVKFF+UiJ zri&${*L1jEC_G5(*CD)E1i<4xqCfh74lFgP$6;!*7z*||A{gvN!XHjg5!DbsQH*rz zNAK0%L|UXGGIx?$DxviSG1=#v-dZKv-i7&r5Py&OTe#&0TeWY%Je|z+*u0@7z}q6x zdxWE^xc3bTcbB21%~@g^MC=!_V9&yEWnpS2?4N*&C2}Z zhtxds$(5)`^TJKF4 zUtPqOePvln-}Vw=7hv9iBW29PL4tOR@mgr9h+yqUF=T4y1^8>tu~_G=HBD>2R9>*x z&A8;Lgy)uBeCThEtMkr>{iP+x4yWb`oz(r$n{e_SF&8fE7Y*=9r5FzT_KOf{pVue@ z{CD4_i(0UI&!tXl`Gikn9!uTXjPrNg6zH~)hF8w_aqB#Y-^(VVp@>IGfzNBie$9Sb zSeUkDiO@5a2roW_E5GhN;i(^oKDJP)P{6WO%zzndM9g;c`vQZ-c{~_0_VD3gkvtlX zEf*(QHQg18Wzh&D6wP8fbTE3Cn@|a&;jpq!49P0O;yC;(M@OzgM~=h1a|ow{S0bkX z-TFavb8_-3^x{foo>GHqlqieP>8q3&__|*7cdv4MCVLJJt`;?5E#MiN?`M1;>b6qE z-a?s$ID6V9M5j5FSxGO{UsEOzRjI}=qBUrR3d~nxsYJP7i9toGu~dpm%~QgZ2T;Lk z{K6K7t%{LfjO8jUIgWqqaUa&sQvQe>s-1^*gH;@!llsfF4J*WD2FB0CaK>af_Y)CS zbK&>~F}VG(<1WabUt?}rFSus^3@gBLR@k*e|KRbSw+^n5a+k;kjBHcD zJVFfZ{WOdNJ5$2QqtUSH3SS9!RoIQVAk$D+npqWwf$_;A0`5*0OW5~-;W)#3CR)d^ z<7!4xxlm?T8i9=xJ-#=^<++Z9bwuUaFl7-2SNkFnM;}EU(KHd)GYMnnxZfC=eck)? zLwix^H8uknwV>HPKRfDShZ3Nfr90t z99rKO1&&ZABe*|(hiD}JTBxiOd5&m?A3c4EZ_rNGiDQga%Y^SI{9#E3W?ISe%*I`wzbU=L|9rZ;HE{+6JP-xQ*uVui>? z)r$|XKRUxUoLR_yduq}*a%D{)(sD--?y7W>_c@oBCMafE&MlJb)Ks!Q&dhf)TCeGOeu@SwHU+8d7u zv(1uXq4@6uKA(wUnt7e5VSOUTrOcjQRZ>|syA&*s3rqVy@g&v_n^sy;l`eN357F!= sgq^qiT{zAPvvsvyv!BJ2Lo=TfP2AP+#(!gk(SJ{DH~XrrrAw&)2aB{0oB#j-