NuCache+Scope - tests

src/Umbraco.Core/Persistence/UmbracoDatabaseFactory.cs

@@ -224,7 +224,7 @@ namespace Umbraco.Core.Persistence
         // ensures that the database is configured, else throws
         private void EnsureConfigured()
         {
-            using (new ReadLock(_lock))
+            using (new ReadLock(_lock)) // fixme - bad, allocations!
             {
                 if (Configured == false)
                     throw new InvalidOperationException("Not configured.");

src/Umbraco.Core/Services/Changes/ContentTypeChangeTypes.cs

@@ -6,8 +6,9 @@ namespace Umbraco.Core.Services.Changes
     public enum ContentTypeChangeTypes : byte
     {
         None = 0,
-        RefreshMain = 1, // changed, impacts content (adding ppty or composition does NOT)
-        RefreshOther = 2, // changed, other changes
-        Remove = 4 // item type has been removed
+        Create = 1, // item type has been created, no impact
+        RefreshMain = 2, // changed, impacts content (adding ppty or composition does NOT)
+        RefreshOther = 4, // changed, other changes
+        Remove = 8 // item type has been removed
     }
 }

src/Umbraco.Core/Services/ContentTypeServiceBaseOfTRepositoryTItemTService.cs

@@ -125,7 +125,7 @@ namespace Umbraco.Core.Services
                 var isNewContentType = dirty.WasPropertyDirty("HasIdentity");
                 if (isNewContentType)
                 {
-                    AddChange(changes, contentType, ContentTypeChangeTypes.RefreshOther);
+                    AddChange(changes, contentType, ContentTypeChangeTypes.Create);
                     continue;
                 }
 

src/Umbraco.Tests/Integration/ContentEventsTests.cs

@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using LightInject;
-using Moq;
 using NUnit.Framework;
 using Umbraco.Core;
 using Umbraco.Core.Cache;
@@ -13,7 +12,6 @@ using Umbraco.Core.Persistence.Repositories;
 using Umbraco.Core.Sync;
 using Umbraco.Tests.Cache.DistributedCache;
 using Umbraco.Tests.Services;
-using Umbraco.Tests.TestHelpers;
 using Umbraco.Tests.TestHelpers.Entities;
 using Umbraco.Tests.Testing;
 using Umbraco.Web.Cache;

src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs

@@ -0,0 +1,136 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Web.Routing;
+using LightInject;
+using Moq;
+using NUnit.Framework;
+using Umbraco.Core;
+using Umbraco.Core.Cache;
+using Umbraco.Core.Composing;
+using Umbraco.Core.Configuration.UmbracoSettings;
+using Umbraco.Core.Models;
+using Umbraco.Core.Sync;
+using Umbraco.Tests.TestHelpers;
+using Umbraco.Tests.Testing;
+using Umbraco.Web;
+using Umbraco.Web.Cache;
+using Umbraco.Web.PublishedCache;
+using Umbraco.Web.PublishedCache.NuCache;
+using Umbraco.Web.Routing;
+using Umbraco.Web.Security;
+
+namespace Umbraco.Tests.Scoping
+{
+    [TestFixture]
+    [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest, FacadeServiceRepositoryEvents = true)]
+    public class ScopedNuCacheTests : TestWithDatabaseBase
+    {
+        private CacheRefresherComponent _cacheRefresher;
+
+        protected override void Compose()
+        {
+            base.Compose();
+
+            // the cache refresher component needs to trigger to refresh caches
+            // but then, it requires a lot of plumbing ;(
+            // fixme - and we cannot inject a DistributedCache yet
+            // so doing all this mess
+            Container.RegisterSingleton<IServerMessenger, ScopedXmlTests.LocalServerMessenger>();
+            Container.RegisterSingleton(f => Mock.Of<IServerRegistrar>());
+            Container.RegisterCollectionBuilder<CacheRefresherCollectionBuilder>()
+                .Add(f => f.TryGetInstance<TypeLoader>().GetCacheRefreshers());
+        }
+
+        public override void TearDown()
+        {
+            base.TearDown();
+
+            _cacheRefresher?.Unbind();
+            _cacheRefresher = null;
+
+            //_onPublishedAssertAction = null;
+            //ContentService.Published -= OnPublishedAssert;
+        }
+
+        protected override IFacadeService CreateFacadeService()
+        {
+            var options = new FacadeService.Options { IgnoreLocalDb = true };
+            var facadeAccessor = new UmbracoContextFacadeAccessor(Umbraco.Web.Composing.Current.UmbracoContextAccessor);
+            var runtimeStateMock = new Mock<IRuntimeState>();
+            runtimeStateMock.Setup(x => x.Level).Returns(() => RuntimeLevel.Run);
+
+            return new FacadeService(
+                options,
+                null,
+                runtimeStateMock.Object,
+                ServiceContext,
+                UowProvider,
+                facadeAccessor,
+                Logger,
+                ScopeProvider);
+        }
+
+        protected UmbracoContext GetUmbracoContextNu(string url, int templateId = 1234, RouteData routeData = null, bool setSingleton = false, IUmbracoSettingsSection umbracoSettings = null, IEnumerable<IUrlProvider> urlProviders = null)
+        {
+            // ensure we have a FacadeService
+            var service = FacadeService as FacadeService;
+
+            var httpContext = GetHttpContextFactory(url, routeData).HttpContext;
+
+            var umbracoContext = new UmbracoContext(
+                httpContext,
+                service,
+                new WebSecurity(httpContext, Current.Services.UserService),
+                umbracoSettings ?? SettingsForTests.GetDefault(),
+                urlProviders ?? Enumerable.Empty<IUrlProvider>());
+
+            if (setSingleton)
+                Umbraco.Web.Composing.Current.UmbracoContextAccessor.UmbracoContext = umbracoContext;
+
+            return umbracoContext;
+        }
+
+        [TestCase(true)]
+        [TestCase(false)]
+        public void WipTest(bool complete)
+        {
+            var umbracoContext = GetUmbracoContextNu("http://example.com/", setSingleton: true);
+
+            // wire cache refresher
+            _cacheRefresher = new CacheRefresherComponent(true);
+            _cacheRefresher.Initialize(new DistributedCache());
+
+            // create document type, document
+            var contentType = new ContentType(-1) { Alias = "CustomDocument", Name = "Custom Document" };
+            Current.Services.ContentTypeService.Save(contentType);
+            var item = new Content("name", -1, contentType);
+
+            using (var scope = ScopeProvider.CreateScope())
+            {
+                Current.Services.ContentService.SaveAndPublishWithStatus(item);
+                item.Name = "changed";
+                Current.Services.ContentService.SaveAndPublishWithStatus(item);
+
+                if (complete)
+                    scope.Complete();
+            }
+
+            // fixme - some exceptions are badly swallowed by the scope 'robust exit'?
+            // fixme - the plumbing of 'other' content types is badly borked
+
+            var x = umbracoContext.ContentCache.GetById(item.Id);
+
+            if (complete)
+            {
+                Assert.IsNotNull(x);
+                Assert.AreEqual("changed", x.Name);
+            }
+            else
+            {
+                Assert.IsNull(x);
+            }
+
+            // fixme - should do more tests & ensure it's all consistent even after rollback
+        }
+    }
+}

src/Umbraco.Tests/TestHelpers/TestWithDatabaseBase.cs

@@ -46,7 +46,6 @@ namespace Umbraco.Tests.TestHelpers
     public abstract class TestWithDatabaseBase : UmbracoTestBase
     {
         private CacheHelper _disabledCacheHelper;
-        private IFacadeService _facadeService;
 
         private string _databasePath;
         private static byte[] _databaseBytes;
@@ -76,7 +75,7 @@ namespace Umbraco.Tests.TestHelpers
             base.Compose();
 
             Container.Register<ISqlSyntaxProvider, SqlCeSyntaxProvider>();
-            Container.Register(factory => _facadeService);
+            Container.Register(factory => FacadeService);
 
             Container.GetInstance<PropertyEditorCollectionBuilder>()
                 .Clear()
@@ -115,8 +114,8 @@ namespace Umbraco.Tests.TestHelpers
                 AppDomain.CurrentDomain.SetData("DataDirectory", null);
 
                 // make sure we dispose of the service to unbind events
-                _facadeService?.Dispose();
-                _facadeService = null;
+                FacadeService?.Dispose();
+                FacadeService = null;
             }
             finally
             {
@@ -227,6 +226,8 @@ namespace Umbraco.Tests.TestHelpers
             }
         }
 
+        protected IFacadeService FacadeService { get; set; }
+
         protected override void Initialize() // fixme - should NOT be here!
         {
             base.Initialize();
@@ -234,37 +235,42 @@ namespace Umbraco.Tests.TestHelpers
             CreateAndInitializeDatabase();
 
             // ensure we have a FacadeService
-            if (_facadeService == null)
+            if (FacadeService == null)
             {
-                var cache = new NullCacheProvider();
-
-                ContentTypesCache = new PublishedContentTypeCache(
-                    Current.Services.ContentTypeService,
-                    Current.Services.MediaTypeService,
-                    Current.Services.MemberTypeService,
-                    Current.Logger);
-
-                // testing=true so XmlStore will not use the file nor the database
-                //var facadeAccessor = new TestFacadeAccessor();
-                var facadeAccessor = new UmbracoContextFacadeAccessor(Umbraco.Web.Composing.Current.UmbracoContextAccessor);
-                var service = new FacadeService(
-                    Current.Services,
-                    (ScopeProvider) Current.ScopeProvider,
-                    UowProvider,
-                    cache, facadeAccessor, Current.Logger, ContentTypesCache, null, true, Options.FacadeServiceRepositoryEvents);
-
-                // initialize PublishedCacheService content with an Xml source
-                service.XmlStore.GetXmlDocument = () =>
-                {
-                    var doc = new XmlDocument();
-                    doc.LoadXml(GetXmlContent(0));
-                    return doc;
-                };
-
-                _facadeService = service;
+                FacadeService = CreateFacadeService();
             }
         }
 
+        protected virtual IFacadeService CreateFacadeService()
+        {
+            var cache = new NullCacheProvider();
+
+            ContentTypesCache = new PublishedContentTypeCache(
+                Current.Services.ContentTypeService,
+                Current.Services.MediaTypeService,
+                Current.Services.MemberTypeService,
+                Current.Logger);
+
+            // testing=true so XmlStore will not use the file nor the database
+            //var facadeAccessor = new TestFacadeAccessor();
+            var facadeAccessor = new UmbracoContextFacadeAccessor(Umbraco.Web.Composing.Current.UmbracoContextAccessor);
+            var service = new FacadeService(
+                Current.Services,
+                (ScopeProvider) Current.ScopeProvider,
+                UowProvider,
+                cache, facadeAccessor, Current.Logger, ContentTypesCache, null, true, Options.FacadeServiceRepositoryEvents);
+
+            // initialize PublishedCacheService content with an Xml source
+            service.XmlStore.GetXmlDocument = () =>
+            {
+                var doc = new XmlDocument();
+                doc.LoadXml(GetXmlContent(0));
+                return doc;
+            };
+
+            return service;
+        }
+
         /// <summary>
         /// Creates the tables and data for the database
         /// </summary>
@@ -331,7 +337,7 @@ namespace Umbraco.Tests.TestHelpers
         protected UmbracoContext GetUmbracoContext(string url, int templateId = 1234, RouteData routeData = null, bool setSingleton = false, IUmbracoSettingsSection umbracoSettings = null, IEnumerable<IUrlProvider> urlProviders = null)
         {
             // ensure we have a PublishedCachesService
-            var service = _facadeService as FacadeService;
+            var service = FacadeService as FacadeService;
             if (service == null)
                 throw new Exception("Not a proper XmlPublishedCache.PublishedCachesService.");
 

src/Umbraco.Tests/Umbraco.Tests.csproj

@@ -217,6 +217,7 @@
     <Compile Include="Scoping\ScopedXmlTests.cs" />
     <Compile Include="Scoping\ScopeEventDispatcherTests.cs" />
     <Compile Include="Scoping\ScopeFileSystemsTests.cs" />
+    <Compile Include="Scoping\ScopedNuCacheTests.cs" />
     <Compile Include="Scoping\ScopeTests.cs" />
     <Compile Include="Services\CachedDataTypeServiceTests.cs" />
     <Compile Include="TestHelpers\ConsoleLogger.cs" />

src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs

@@ -235,6 +235,55 @@ namespace Umbraco.Web.PublishedCache.NuCache
 
         #region Content types
 
+        public void NewContentTypes(IEnumerable<PublishedContentType> types)
+        {
+            var lockInfo = new WriteLockInfo();
+            try
+            {
+                Lock(lockInfo);
+
+                foreach (var type in types)
+                {
+                    SetValueLocked(_contentTypesById, type.Id, type);
+                    SetValueLocked(_contentTypesByAlias, type.Alias, type);
+                }
+            }
+            finally
+            {
+                Release(lockInfo);
+            }
+        }
+
+        public void UpdateContentTypes(IEnumerable<PublishedContentType> types)
+        {
+            var lockInfo = new WriteLockInfo();
+            try
+            {
+                Lock(lockInfo);
+
+                var index = types.ToDictionary(x => x.Id, x => x);
+
+                foreach (var type in index.Values)
+                {
+                    SetValueLocked(_contentTypesById, type.Id, type);
+                    SetValueLocked(_contentTypesByAlias, type.Alias, type);
+                }
+
+                foreach (var link in _contentNodes.Values)
+                {
+                    var node = link.Value;
+                    if (node == null) continue;
+                    var contentTypeId = node.ContentType.Id;
+                    if (index.TryGetValue(contentTypeId, out PublishedContentType contentType) == false) continue;
+                    SetValueLocked(_contentNodes, node.Id, new ContentNode(node, contentType, _facadeAccessor));
+                }
+            }
+            finally
+            {
+                Release(lockInfo);
+            }
+        }
+
         public void UpdateContentTypes(IEnumerable<int> removedIds, IEnumerable<PublishedContentType> refreshedTypes, IEnumerable<ContentNodeKit> kits)
         {
             var removedIdsA = removedIds?.ToArray() ?? Array.Empty<int>();

src/Umbraco.Web/PublishedCache/NuCache/FacadeService.cs

@@ -152,7 +152,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
 
                 try
                 {
-                    if (_localDbExists)
+                    if (_localDbExists) // fixme?
                     {
                         LockAndLoadContent(LoadContentFromLocalDbLocked);
                         LockAndLoadMedia(LoadMediaFromLocalDbLocked);
@@ -196,6 +196,29 @@ namespace Umbraco.Web.PublishedCache.NuCache
             MemberTypeService.UowRefreshedEntity += OnMemberTypeRefreshedEntity;
         }
 
+        private void TearDownRepositoryEvents()
+        {
+            ContentRepository.UowRemovingEntity -= OnContentRemovingEntity;
+            //ContentRepository.RemovedVersion -= OnContentRemovedVersion;
+            ContentRepository.UowRefreshedEntity -= OnContentRefreshedEntity;
+            MediaRepository.UowRemovingEntity -= OnMediaRemovingEntity;
+            //MediaRepository.RemovedVersion -= OnMediaRemovedVersion;
+            MediaRepository.UowRefreshedEntity -= OnMediaRefreshedEntity;
+            MemberRepository.UowRemovingEntity -= OnMemberRemovingEntity;
+            //MemberRepository.RemovedVersion -= OnMemberRemovedVersion;
+            MemberRepository.UowRefreshedEntity -= OnMemberRefreshedEntity;
+
+            ContentTypeService.UowRefreshedEntity -= OnContentTypeRefreshedEntity;
+            MediaTypeService.UowRefreshedEntity -= OnMediaTypeRefreshedEntity;
+            MemberTypeService.UowRefreshedEntity -= OnMemberTypeRefreshedEntity;
+        }
+
+        public override void Dispose()
+        {
+            TearDownRepositoryEvents();
+            base.Dispose();
+        }
+
         public class Options
         {
             // indicates that the facade cache should reuse the application request cache
@@ -672,45 +695,45 @@ namespace Umbraco.Web.PublishedCache.NuCache
             foreach (var payload in payloads)
                 _logger.Debug<XmlStore>($"Notified {payload.ChangeTypes} for {payload.ItemType} {payload.Id}");
 
-            var removedIds = payloads
-                .Where(x => x.ItemType == typeof(IContentType).Name && x.ChangeTypes.HasType(ContentTypeChangeTypes.Remove))
-                .Select(x => x.Id)
-                .ToArray();
-
-            var refreshedIds = payloads
-                .Where(x => x.ItemType == typeof(IContentType).Name && x.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshMain))
-                .Select(x => x.Id)
-                .ToArray();
-
-            if (removedIds.Length > 0 || refreshedIds.Length > 0)
-                using (_contentStore.GetWriter(_scopeProvider))
-                {
-                    // ReSharper disable AccessToModifiedClosure
-                    RefreshContentTypesLocked(removedIds, refreshedIds);
-                    // ReSharper restore AccessToModifiedClosure
-                }
-
-            // same for media cache
-
-            removedIds = payloads
-                .Where(x => x.ItemType == typeof(IMediaType).Name && x.ChangeTypes.HasType(ContentTypeChangeTypes.Remove))
-                .Select(x => x.Id)
-                .ToArray();
-
-            refreshedIds = payloads
-                .Where(x => x.ItemType == typeof(IMediaType).Name && x.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshMain))
-                .Select(x => x.Id)
-                .ToArray();
-
-            if (removedIds.Length > 0 || refreshedIds.Length > 0)
-                using (_mediaStore.GetWriter(_scopeProvider))
-                {
-                    RefreshMediaTypesLocked(removedIds, refreshedIds);
-                }
+            Notify<IContentType>(_contentStore, payloads, RefreshContentTypesLocked);
+            Notify<IMediaType>(_mediaStore, payloads, RefreshMediaTypesLocked);
 
             ((Facade)CurrentFacade).Resync();
         }
 
+        private void Notify<T>(ContentStore store, ContentTypeCacheRefresher.JsonPayload[] payloads, Action<IEnumerable<int>, IEnumerable<int>, IEnumerable<int>, IEnumerable<int>> action)
+        {
+            var nameOfT = typeof (T).Name;
+
+            var removedIds = new List<int>();
+            var refreshedIds = new List<int>();
+            var otherIds = new List<int>();
+            var newIds = new List<int>();
+
+            foreach (var payload in payloads)
+            {
+                if (payload.ItemType != nameOfT) continue;
+
+                if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.Remove))
+                    removedIds.Add(payload.Id);
+                else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshMain))
+                    refreshedIds.Add(payload.Id);
+                else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshOther))
+                    otherIds.Add(payload.Id);
+                else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.Create))
+                    newIds.Add(payload.Id);
+            }
+
+            if (removedIds.Count == 0 && refreshedIds.Count == 0 && otherIds.Count == 0) return;
+
+            using (store.GetWriter(_scopeProvider))
+            {
+                // ReSharper disable AccessToModifiedClosure
+                action(removedIds, refreshedIds, otherIds, newIds);
+                // ReSharper restore AccessToModifiedClosure
+            }
+        }
+
         public override void Notify(DataTypeCacheRefresher.JsonPayload[] payloads)
         {
             // no cache, nothing we can do
@@ -835,7 +858,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
             return contentType == null ? null : new PublishedContentType(itemType, contentType);
         }
 
-        private void RefreshContentTypesLocked(IEnumerable<int> removedIds, IEnumerable<int> refreshedIds)
+        private void RefreshContentTypesLocked(IEnumerable<int> removedIds, IEnumerable<int> refreshedIds, IEnumerable<int> otherIds, IEnumerable<int> newIds)
         {
             // locks:
             // content (and content types) are read-locked while reading content
@@ -853,11 +876,13 @@ namespace Umbraco.Web.PublishedCache.NuCache
                 var typesA = CreateContentTypes(PublishedItemType.Content, refreshedIdsA).ToArray();
                 var kits = _dataSource.GetTypeContentSources(uow, refreshedIdsA);
                 _contentStore.UpdateContentTypes(removedIds, typesA, kits);
+                _contentStore.UpdateContentTypes(CreateContentTypes(PublishedItemType.Content, otherIds.ToArray()).ToArray());
+                _contentStore.NewContentTypes(CreateContentTypes(PublishedItemType.Content, newIds.ToArray()).ToArray());
                 uow.Complete();
             }
         }
 
-        private void RefreshMediaTypesLocked(IEnumerable<int> removedIds, IEnumerable<int> refreshedIds)
+        private void RefreshMediaTypesLocked(IEnumerable<int> removedIds, IEnumerable<int> refreshedIds, IEnumerable<int> otherIds, IEnumerable<int> newIds)
         {
             // locks:
             // media (and content types) are read-locked while reading media
@@ -875,6 +900,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
                 var typesA = CreateContentTypes(PublishedItemType.Media, refreshedIdsA).ToArray();
                 var kits = _dataSource.GetTypeMediaSources(uow, refreshedIdsA);
                 _mediaStore.UpdateContentTypes(removedIds, typesA, kits);
+                _mediaStore.UpdateContentTypes(CreateContentTypes(PublishedItemType.Media, otherIds.ToArray()).ToArray());
+                _mediaStore.NewContentTypes(CreateContentTypes(PublishedItemType.Media, newIds.ToArray()).ToArray());
                 uow.Complete();
             }
         }

src/Umbraco.Web/PublishedCache/XmlPublishedCache/XmlStore.cs

@@ -1634,7 +1634,7 @@ ORDER BY umbracoNode.level, umbracoNode.sortOrder";
         private void OnContentTypeRefreshedEntity(IContentTypeService sender, ContentTypeChange<IContentType>.EventArgs args)
         {
             const ContentTypeChangeTypes types // only for those that have been refreshed
-                = ContentTypeChangeTypes.RefreshMain | ContentTypeChangeTypes.RefreshOther;
+                = ContentTypeChangeTypes.RefreshMain | ContentTypeChangeTypes.RefreshOther | ContentTypeChangeTypes.Create;
             var contentTypeIds = args.Changes.Where(x => x.ChangeTypes.HasTypesAny(types)).Select(x => x.Item.Id).ToArray();
             if (contentTypeIds.Any())
                 RebuildContentAndPreviewXml(contentTypeIds: contentTypeIds);
@@ -1643,7 +1643,7 @@ ORDER BY umbracoNode.level, umbracoNode.sortOrder";
         private void OnMediaTypeRefreshedEntity(IMediaTypeService sender, ContentTypeChange<IMediaType>.EventArgs args)
         {
             const ContentTypeChangeTypes types // only for those that have been refreshed
-                = ContentTypeChangeTypes.RefreshMain | ContentTypeChangeTypes.RefreshOther;
+                = ContentTypeChangeTypes.RefreshMain | ContentTypeChangeTypes.RefreshOther | ContentTypeChangeTypes.Create;
             var mediaTypeIds = args.Changes.Where(x => x.ChangeTypes.HasTypesAny(types)).Select(x => x.Item.Id).ToArray();
             if (mediaTypeIds.Any())
                 RebuildMediaXml(contentTypeIds: mediaTypeIds);
@@ -1652,7 +1652,7 @@ ORDER BY umbracoNode.level, umbracoNode.sortOrder";
         private void OnMemberTypeRefreshedEntity(IMemberTypeService sender, ContentTypeChange<IMemberType>.EventArgs args)
         {
             const ContentTypeChangeTypes types // only for those that have been refreshed
-                = ContentTypeChangeTypes.RefreshMain | ContentTypeChangeTypes.RefreshOther;
+                = ContentTypeChangeTypes.RefreshMain | ContentTypeChangeTypes.RefreshOther | ContentTypeChangeTypes.Create;
             var memberTypeIds = args.Changes.Where(x => x.ChangeTypes.HasTypesAny(types)).Select(x => x.Item.Id).ToArray();
             if (memberTypeIds.Any())
                 RebuildMemberXml(contentTypeIds: memberTypeIds);