Cache.cs

using System;
using System.IO;
using System.Text;
using System.Collections;
using System.Collections.Specialized;
using System.Security.Cryptography;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Formatters.Binary;


namespace Common {
        
        [Serializable]
        public class CacheIndex : ListDictionary {
                private readonly string BASE_PATH;
                private readonly string EXT;
                private long maxCacheSize;

                public CacheIndex(string basePath, string ext, long maxCacheSize) {
                        this.BASE_PATH = basePath;
                        this.EXT = ext;
                        this.maxCacheSize = maxCacheSize;
                        UpdateFromDisk();       
                }

                public static CacheIndex Deserialise(byte[] srcCompressed) {
                        MemoryStream compressedStream = new MemoryStream(srcCompressed);
                        byte[] srcUncompressed = EncodedData.StaticUnzip(ref compressedStream);
                        compressedStream.Close();
                        IFormatter formatter = new BinaryFormatter(); 
                        return (CacheIndex)formatter.Deserialize(new MemoryStream(srcUncompressed));
                }

                public byte[] Serialise() {
                        IFormatter formatter = new BinaryFormatter();
                        MemoryStream bufferStream = new MemoryStream();
                        formatter.Serialize(bufferStream, this);
                        byte[] result = EncodedData.StaticZip(bufferStream);
                        bufferStream.Close();
                        return result;
                }

                private void UpdateFromDisk() {
                        lock (SyncRoot) {
                                Console.Write("Populating Cache Index... ");
                                string[] searchResult = Directory.GetFiles(BASE_PATH, "*" + EXT);
                                foreach(string filename in searchResult) {
                                        FileStream fs = File.OpenRead(filename);
                                        CacheIndexEntry cie = new CacheIndexEntry(Cache.ChkFromString(Path.GetFileNameWithoutExtension(filename)), Path.GetFileNameWithoutExtension(filename), (int)fs.Length, CacheEntry.DeserialiseDate(fs));
                                        fs.Close();
                                        this.Add(cie);
                                }
                                Console.WriteLine("Done.");
                        }
                }
                public CacheIndexEntry this[byte[] chk] {
                        get { 
                                lock(SyncRoot) {
                                        return (CacheIndexEntry)base[Cache.StringFromChk(chk)]; 
                                }
                        }
                        set { 
                                lock(SyncRoot) {
                                        base[Cache.StringFromChk(chk)] = value; 
                                }
                        }
                }

                public bool Contains(byte[] chk){
                        lock (SyncRoot) {
                                return this.Contains(Cache.StringFromChk(chk));
                        }
                }

                public void Remove(byte[] chk) {
                        lock (SyncRoot) {
                                base.Remove(Cache.StringFromChk(chk));
                        }
                }

                public void Add(CacheIndexEntry cie) {
                        lock (SyncRoot) {
                                this.Add(Cache.StringFromChk(cie.chk), cie);
                        }
                }

                public CacheIndexEntry LeastRecentlyUsed() {
                        lock (SyncRoot) {
                                CacheIndexEntry evictionCandidate = null;
                                if (this.Count != 0) {
                                        DateTime minLRU = DateTime.MaxValue;
                                        foreach(object o in this) {
                                                CacheIndexEntry cie = (CacheIndexEntry)((DictionaryEntry)o).Value;
                                                if (cie.lastUsed <= minLRU) {
                                                        minLRU = cie.lastUsed;
                                                        evictionCandidate = cie;
                                                }
                                        }
                                }
                                return evictionCandidate;
                        }
                }

                public long GetSpaceRemaining() {
                        lock (SyncRoot) {
                                long totalSize = 0;
                                foreach(object o in this.Values) {
                                        CacheIndexEntry cie = (CacheIndexEntry)o;
                                        totalSize += cie.size;
                                }
                                return maxCacheSize - totalSize;
                        }
                }
        }

        /*************************************************************************/
        [Serializable]
        public class CacheIndexEntry {
                public byte[] chk;
                public string filename;
                public int size;
                public DateTime lastUsed;

                public CacheIndexEntry(byte[] chk, string filename, int size, DateTime lastUsed) {
                        this.chk = chk;
                        this.filename = filename;
                        this.size = size;
                        this.lastUsed = lastUsed;
                }
        }

        /*************************************************************************/
        public class Cache {
                CommonSettings settings;  
                readonly string BASE_PATH;  
                readonly string EXT;
                
                SHA1Managed sha1;
                CacheIndex index;
                Object SyncRoot = new Object();
                long maxSize;
                long spaceUsed;
                internal CacheManager cacheManager;

                public Cache(long maxSize, CacheManager cacheManager) {
                        this.cacheManager = cacheManager;
                        this.maxSize = maxSize;
                        settings = new CommonSettings();
                        BASE_PATH = settings.CacheLocation;
                        EXT = settings.CacheFileExtension;
                        sha1 = new SHA1Managed();
                        index = new CacheIndex(BASE_PATH, EXT, maxSize);
                        spaceUsed = CheckSpaceUsed();
                }

                long CheckSpaceUsed() {
                        lock (SyncRoot) {
                                DirectoryInfo dirInfo = new DirectoryInfo(BASE_PATH);
                                long totalSize = 0;
                                FileSystemInfo[] fiArr = dirInfo.GetFileSystemInfos("*" + EXT);
                                foreach (FileInfo fi in fiArr) {
                                        totalSize += fi.Length;
                                }
                                Console.WriteLine("Cache files take up {0} bytes on disk.", totalSize);
                                return totalSize;
                        }
                }

                public byte[] Put(HTTPBody body) {
                        lock (SyncRoot) {
                                byte[] chk = null; 
                                if (body != null) {
                                        chk = sha1.ComputeHash(body.data);
                                        /* check to see if CHK already exists:
                                         *  1) Check index. If match, double check file existance(if no exist,
                                         *  delete from index), then return. Error?
                                         *  2) check filesystem itself. If match, add it to the index, return. Error? */
                                        string prefix = "";
                                        bool alreadyExists = false;
                                        if (index.Contains(chk)) {
                                                CacheIndexEntry currentEntry = index[chk];
                                                prefix = currentEntry.filename;
                                                if (File.Exists(BASE_PATH + currentEntry.filename + EXT)) { // all well and good.
                                                        alreadyExists = true;
                                                } else {
                                                        index.Remove(chk); // since it didn't actually exist in the first place.
                                                }
                                        } else { // TODO: Tidy this up
                                                prefix = Cache.StringFromChk(chk);
                                                if (File.Exists(BASE_PATH + prefix + EXT)) { // all well and good.
                                                        alreadyExists = true;
                                                }
                                        }
                                        if (!alreadyExists) {
                                                // create the cache entry
                                                CacheEntry ce = new CacheEntry(body);
                                                byte[] buffer = ce.Serialise();
                                                if (buffer.Length + spaceUsed > maxSize) {
                                                        Purge(maxSize - buffer.Length); // purge cache until at most maxSize - bufferLength remains (ie enough for buffer length
                                                }
                                                // create the index entry (CHK, Filename, Size, LastUsed)
                                                CacheIndexEntry cie = new CacheIndexEntry(chk, prefix, buffer.Length, DateTime.Now);
                                                // create the file
                                                FileStream newEntry = File.Create(BASE_PATH + prefix + EXT);
                                                // write the serialised cache entry             
                                                newEntry.Write(buffer, 0, buffer.Length);
                                                // close everything that needs to be closed                                     
                                                newEntry.Close();
                                                // write to the index
                                                index.Add(cie);
                                                spaceUsed += cie.size;
                                        } 
                                } // end if body.data != null;
                                return chk;
                        } // unlockl
                }

                public HTTPBody Get(byte[] chk) {
                        lock (SyncRoot) {
                                HTTPBody result;
                                CacheIndexEntry indexEntry = index[chk];
                                FileStream fs;
                                try {
                                        fs = File.Open(BASE_PATH + indexEntry.filename + EXT, FileMode.Open, FileAccess.ReadWrite);
                                } catch (Exception ex) {
                                        throw new CacheNotFoundException(String.Format("Unable to find {0}", Cache.StringFromChk(chk)), ex);
                                }
                                // reads entry
                                /* As part of new 1.1 eviction policy, we don't want to update LRU on read, only on write
                                 * Thus LRU becomes more like a FIFO. Except it's still called LRU because I am too lazy
                                 * to rename variables
                                 * */
                                CacheEntry cacheEntry = new CacheEntry(fs);
                                result = cacheEntry.data;
//                              cacheEntry.lastUsed = DateTime.Now;
//                              fs.Seek(0, SeekOrigin.Begin);
//                              byte[] dateBuffer = cacheEntry.SerialiseDate();
//                              fs.Write(dateBuffer, 0, dateBuffer.Length);
                                fs.Close();
//                              indexEntry.lastUsed = cacheEntry.lastUsed;
//                              index[chk] = indexEntry;
                                return result;
                        }
                }

                public void Delete(byte[] chk) {
                        lock (SyncRoot) {
                                CacheIndexEntry indexEntry = index[chk];
                                if (File.Exists(BASE_PATH + indexEntry.filename + EXT)) {
                                        File.Delete(BASE_PATH + indexEntry.filename + EXT);
                                }
                                index.Remove(chk);
                                spaceUsed -= indexEntry.size;
                        }
                }

                public void Purge(long targetUsage) {
                        lock (SyncRoot) {
                                long bytesRemaining = spaceUsed - targetUsage;
                                while (bytesRemaining > 0) {
                                        byte[] evictionTargetCHK = index.LeastRecentlyUsed().chk;
                                        Delete(evictionTargetCHK);
                                        cacheManager.uriMapper.RemoveAssociatedUris(evictionTargetCHK);
                                        bytesRemaining = spaceUsed - targetUsage;
                                }
                        }
                }

                public bool Contains(byte[] chk) {
                        lock (SyncRoot) {
                                return File.Exists(BASE_PATH + Cache.StringFromChk(chk) + EXT);
                        }
                }
                public void Delete(string chk) {
                        Delete(Cache.ChkFromString(chk));
                }

                public static string StringFromChk(byte[] chk) {
                        string result = "";
                        foreach (byte b in chk) {
                                result += getHex(b);
                        }
                        return result;
                }

                public static string getHex(byte src) {
                        return src.ToString("X2");
                }

                public static byte[] ChkFromString(string src) {
                        byte[] result = new byte[20];
                        if (src.Length == 40) {
                                string chunk;
                                for (int i = 0; i < src.Length; i+=2) {
                                        chunk = src.Substring(i, 2);
                                        result[i/2] = Byte.Parse(chunk, System.Globalization.NumberStyles.HexNumber);
                                }
                        } else {
                                throw new ApplicationException("Expected string of length 40. Got length " + src.Length);
                        }
                        return result;
                }

                public CacheIndex GetCacheIndex() {
                        return index;
                }
        }

        /*************************************************************************/
        public class CacheEntry {
                public HTTPBody data;
                public DateTime lastUsed;

                public CacheEntry(HTTPBody data) {
                        this.data = data;
                        this.lastUsed = DateTime.MinValue; 
                }

                public CacheEntry(FileStream fs) {
                        byte[] buffer = new byte[fs.Length];
                        fs.Read(buffer, 0, (int)fs.Length);
                        Deserialise(buffer);
                }

                public byte[] Serialise() {
                        byte[] lastUsedBuffer = BitConverter.GetBytes(lastUsed.Ticks);
                        byte[] buffer = new byte[lastUsedBuffer.Length + data.data.Length];
                        Array.Copy(lastUsedBuffer, 0, buffer, 0, lastUsedBuffer.Length);
                        Array.Copy(data.data, 0, buffer, lastUsedBuffer.Length, data.data.Length);
                        return buffer;
                }

                public void Deserialise(byte[] src) {
                        int dateTimeLength = BitConverter.GetBytes(DateTime.Now.Ticks).Length;
                        byte[] bodyBuffer = new byte[src.Length - dateTimeLength];
                        Array.Copy(src, dateTimeLength, bodyBuffer, 0, src.Length - dateTimeLength);
                        
                        this.data = new HTTPBody(bodyBuffer);
                        this.lastUsed = new DateTime(BitConverter.ToInt64(src, 0));
                }

                public static DateTime DeserialiseDate(FileStream fs) {
                        byte[] dtBuffer = new byte[8]; // a long/date is 8 bytes (64 bits)
                        fs.Seek(0, SeekOrigin.Begin);
                        fs.Read(dtBuffer, 0, dtBuffer.Length);
                        return new DateTime(BitConverter.ToInt64(dtBuffer, 0));
                }

                public byte[] SerialiseDate() {
                        return BitConverter.GetBytes(lastUsed.Ticks);
                }

                public static int SizeOfSerialised(HTTPBody data) {
                        return (BitConverter.GetBytes(DateTime.Now.Ticks).Length + data.data.Length);
                }
                
        }
        
        
        /*************************************************************************/
        public class URIMapper : Hashtable {
                public URIMapper() : base() {}

                public void Add(Uri key, URIMappingEntry data) {
                        lock (this) {
                                base.Add(key, data);
                        }
                }

                public URIMappingEntry this[Uri key] {
                        get { return (URIMappingEntry)base[key];}
                        set { lock (this){
                                          base[key] = value; 
                                  }
                        }
                }

                public void Remove(Uri key) {
                        lock (this) {
                                base.Remove(key);
                        }
                }

                public void RemoveAssociatedUris(byte[] chk) {
                        ArrayList killList = new ArrayList();
                        lock (this) {
                                foreach (DictionaryEntry de in this) {
                                        if (Cache.StringFromChk(((URIMappingEntry)de.Value).CHK) == Cache.StringFromChk(chk)) {
                                                killList.Add(de.Key);
                                        }
                                }
                                foreach (object o in killList) {
                                        this.Remove((Uri)o);
                                }
                        }
                }

                public bool IsFresh(Uri key) { // TODO: Some more complex testing for freshness
                        return !this[key].HasExpired;
                }

                public bool Contains(Uri key) {
                        return base.Contains(key);
                }
        }
        /*************************************************************************/
        public class URIMappingEntry {
                Uri key;
                byte[] chk;
                DateTime creationStamp;
                string headers;

                public URIMappingEntry(Uri key, byte[] chk, string headers) { // is timestamp expiry date, or entry date? Make it entry...
                        this.key = key;
                        this.chk = chk;
                        this.headers = headers;
                        creationStamp = DateTime.Now;
                }

                public bool HasExpired {
                        get { return (DateTime.Now - creationStamp) > TimeSpan.FromMinutes(15); } // all mappings expire after 5 mins.
                }

                public byte[] CHK {
                        get { return chk; }
                }

                public string Headers {
                        get { return headers; }
                }

                public void Freshen(byte[] newChk, string newHeaders) {
                        creationStamp = DateTime.Now;
                        if (headers != newHeaders) {
                                headers = newHeaders;
                        } 
                        if (chk != null && newChk != null && Cache.StringFromChk(chk) != Cache.StringFromChk(newChk)) {
                                chk = newChk;
                        }
                }
        }
        /*************************************************************************/
        public class CacheNotFoundException : ApplicationException {
                public CacheNotFoundException(string message, Exception innerException) : base (message, innerException) {
                }
                public CacheNotFoundException(string message) : base (message) {
                }
                public CacheNotFoundException(): base() {
                }
        }
}

---