swcs/OurUmbraco
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixing up lucene indexes with versions and range queries, should be working now

Browse Source
This commit is contained in:
Shannon
2017-04-27 10:35:16 +10:00
parent 11f9219419
commit baebd73f19
7 changed files with 51 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OurUmbraco.Site/Views/Partials/Projects/ListProjects.cshtml
-1
View File
@@ -55,7 +55,6 @@
var toVersion = new Version(parsedVersion.Major, 999, 999).GetNumericalValue();
var versionFilters = new SearchFilters(BooleanOperation.Or);
versionFilters.Filters.Add(new RangeSearchFilter("num_version", fromVersion, toVersion));
versionFilters.Filters.Add(new RangeSearchFilter("num_compatVersions", fromVersion, toVersion));
filters.Add(versionFilters);
}
}
OurUmbraco.Site/macroScripts/repository-popularpackages.cshtml
-1
View File
@@ -33,7 +33,6 @@
var numericalVersion = parsedVersion.GetNumericalValue();
var versionFilters = new SearchFilters(BooleanOperation.Or);
versionFilters.Filters.Add(new RangeSearchFilter("num_version", 0, numericalVersion));
versionFilters.Filters.Add(new RangeSearchFilter("num_compatVersions", 0, numericalVersion));
filters.Add(versionFilters);
}
OurUmbraco/Our/Api/SearchController.cs
-1
View File
@@ -38,7 +38,6 @@ namespace OurUmbraco.Our.Api
var numericalVersion = parsedVersion.GetNumericalValue();
var versionFilters = new SearchFilters(BooleanOperation.Or);
versionFilters.Filters.Add(new RangeSearchFilter("num_version", 0, numericalVersion));
versionFilters.Filters.Add(new RangeSearchFilter("num_compatVersions", 0, numericalVersion));
filters.Add(versionFilters);
}
}
OurUmbraco/Our/Examine/ProjectNodeIndexDataService.cs
+39 -78
View File
@@ -9,6 +9,7 @@ using Examine.LuceneEngine;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Documents;
using OurUmbraco.Project;
using OurUmbraco.Repository.Services;
using OurUmbraco.Wiki.BusinessLogic;
using umbraco;
using Umbraco.Core;
@@ -34,15 +35,8 @@ namespace OurUmbraco.Our.Examine
var isLive = project.GetPropertyValue<bool>("projectLive");
var isApproved = project.GetPropertyValue<bool>("approved");
var minimumVersionStrict = string.Empty;
var currentFileId = project.GetPropertyValue<int>("file");
if (currentFileId > 0)
{
var currentFile = files.FirstOrDefault(x => x.Id == currentFileId);
if (currentFile != null)
minimumVersionStrict = currentFile.MinimumVersionStrict;
}
var strictPackageFiles = PackageRepositoryService.GetAllStrictSupportedPackageVersions(files);
simpleDataSet.NodeDefinition.NodeId = project.Id;
simpleDataSet.NodeDefinition.Type = indexType;
@@ -54,8 +48,7 @@ namespace OurUmbraco.Our.Examine
simpleDataSet.RowData.Add("nodeTypeAlias", "project");
simpleDataSet.RowData.Add("url", project.Url);
simpleDataSet.RowData.Add("uniqueId", project.GetPropertyValue<string>("packageGuid"));
simpleDataSet.RowData.Add("worksOnUaaS", project.GetPropertyValue<string>("worksOnUaaS"));
simpleDataSet.RowData.Add("minimumVersionStrict", minimumVersionStrict);
simpleDataSet.RowData.Add("worksOnUaaS", project.GetPropertyValue<string>("worksOnUaaS"));
var imageFile = string.Empty;
if (project.HasValue("defaultScreenshotPath"))
@@ -75,37 +68,14 @@ namespace OurUmbraco.Our.Examine
// so we have to do all of this parsing.
var version = project.GetPropertyValue<string>("compatibleVersions") ?? string.Empty;
var cleanedVersions = version.ToLower()
.Replace("nan", "")
.Replace("saved", "")
.Replace("v", "")
.Trim(',')
.Split(',')
//it's stored as an int like 721 (for version 7.2.1)
.Where(x => x.Length <= 3 && x.Length > 0)
//pad it out to 3 digits
.Select(x => x.PadRight(3, '0'))
.Select(x =>
{
int o;
if (int.TryParse(x, out o))
{
//if it ends with '0', that means it's a X.X.X version
// if it does not end with '0', that means that the last 2 digits are the
// Minor part of the version
return x.EndsWith("0")
? string.Format("{0}.{1}.{2}", x[0], x[1], 0)
: string.Format("{0}.{1}.{2}", x[0], x.Substring(1), 0);
}
return null;
})
.Select(x => x.GetFromUmbracoString(reduceToConfigured:false))
.Where(x => x != null);
var cleanedCompatVersions = compatVersions.Select(x => x.Replace("nan", "")
.Replace("saved", "")
.Replace("nan", "")
.Replace("v", "")
.Replace(".x", "")
.Trim(','));
var cleanedCompatVersions = compatVersions
.Select(x => x.GetFromUmbracoString(reduceToConfigured: false))
.Where(x => x != null);
//popularity for sorting number = downloads + karma * 100;
//TODO: Change score so that we take into account:
@@ -136,6 +106,8 @@ namespace OurUmbraco.Our.Examine
//then we index the versions that the project has actually been flagged as compatible against
simpleDataSet.RowData.Add("compatVersions", string.Join(",", cleanedCompatVersions));
simpleDataSet.RowData.Add("minimumVersionStrict", string.Join(",", strictPackageFiles.Select(x => x.MinUmbracoVersion.ToString(3))));
return simpleDataSet;
}
@@ -193,7 +165,7 @@ namespace OurUmbraco.Our.Examine
foreach (var numericalVersion in numericalVersions)
{
//don't store, we're just using this to search
var versionField = new NumericField(fieldName, Field.Store.NO, true).SetLongValue(numericalVersion);
var versionField = new NumericField(fieldName, Field.Store.YES, true).SetLongValue(numericalVersion);
e.Document.Add(versionField);
}
}
@@ -219,50 +191,39 @@ namespace OurUmbraco.Our.Examine
e.Document.Add(new Field("body", e.Fields["body"].StripHtml(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES));
}
//If there is a versions field, we'll split it and index the same field on each version
if (e.Fields.ContainsKey("versions"))
var allVersions = new HashSet<string>();
//each of these contains versions, we want to parse them all into one list and then ensure each of these
//fields are not analyzed (just stored since we dont use them for searching)
var delimitedVersionFields = new[] {"versions", "minimumVersionStrict", "compatVersions"};
foreach (var fieldName in delimitedVersionFields)
{
//split into separate versions
var versions = e.Fields["versions"].Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
AddNumericalVersionValue(e, "num_versions", versions);
//remove the current version field from the lucene doc
e.Document.RemoveField("versions");
foreach (var version in versions)
//If there is a versions field, we'll split it and index the same field on each version
if (e.Fields.ContainsKey(fieldName))
{
//add a 'versions' field for each version (same field name but different values)
//not analyzed, we don't use this for searching
e.Document.Add(new Field("versions", version, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
}
}
//add the minimumVersionStrict as a version too
if (e.Fields.ContainsKey("minimumVersionStrict"))
{
AddNumericalVersionValue(e, "num_versions", new[] {e.Fields["minimumVersionStrict"]});
}
//If there is a compatVersions field, we'll split it and index the same field on each version
if (e.Fields.ContainsKey("compatVersions"))
{
//split into separate versions
var compatVersions = e.Fields["compatVersions"].Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
AddNumericalVersionValue(e, "num_compatVersions", compatVersions);
//remove the current compatVersions field from the lucene doc
e.Document.RemoveField("compatVersions");
foreach (var version in compatVersions)
{
//add a 'compatVersions' field for each compatVersion (same field name but different values)
//not analyzed, we don't use this for searching
e.Document.Add(new Field("compatVersions", version, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
//split into separate versions
var versions = e.Fields[fieldName].Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var version in versions)
{
allVersions.Add(version);
}
//remove the current version field from the lucene doc
e.Document.RemoveField(fieldName);
foreach (var version in versions)
{
//add a 'versions' field for each version (same field name but different values)
//not analyzed, we don't use this for searching
e.Document.Add(new Field(fieldName, version, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
}
}
}
//now add all versions to a numerical field
AddNumericalVersionValue(e, "num_version", allVersions.ToArray());
}
OurUmbraco/Project/Controllers/RepositoryController.cs
-1
View File
@@ -31,7 +31,6 @@ namespace OurUmbraco.Project.Controllers
var numericalVersion = parsedVersion.GetNumericalValue();
var versionFilters = new SearchFilters(BooleanOperation.Or);
versionFilters.Filters.Add(new RangeSearchFilter("num_version", 0, numericalVersion));
versionFilters.Filters.Add(new RangeSearchFilter("num_compatVersions", 0, numericalVersion));
filters.Add(versionFilters);
}
}
OurUmbraco/Project/ProjectExtensions.cs
+1 -1
View File
@@ -23,7 +23,7 @@ namespace OurUmbraco.Project
/// </returns>
public static System.Version GetFromUmbracoString(this string version, bool reduceToConfigured = true)
{
if (string.IsNullOrWhiteSpace(version)) throw new ArgumentException("Value cannot be null or whitespace.", "version");
if (string.IsNullOrWhiteSpace(version)) return null;
//need to clean up this string, it could be all sorts of things
version = version.ToLower()
OurUmbraco/Repository/Services/PackageRepositoryService.cs
+11 -43
View File
@@ -96,13 +96,16 @@ namespace OurUmbraco.Repository.Services
if (version.IsNullOrWhiteSpace() == false)
{
//need to clean up this string, it could be all sorts of things
var parsedVersion = version.GetFromUmbracoString();
var parsedVersion = version.GetFromUmbracoString(reduceToConfigured: false);
if (parsedVersion != null)
{
var numericalVersion = parsedVersion.GetNumericalValue();
var versionFilters = new SearchFilters(BooleanOperation.Or);
versionFilters.Filters.Add(new RangeSearchFilter("num_version", 0, numericalVersion));
versionFilters.Filters.Add(new RangeSearchFilter("num_compatVersions", 0, numericalVersion));
//search for all versions from the current major to the version passed in
var currMajor = new System.Version(parsedVersion.Major, 0, 0).GetNumericalValue();
versionFilters.Filters.Add(new RangeSearchFilter("num_version", currMajor, numericalVersion));
filters.Add(versionFilters);
}
}
@@ -337,7 +340,7 @@ namespace OurUmbraco.Repository.Services
private IEnumerable<System.Version> GetAllFilePackageVersions(IEnumerable<WikiFile> packages)
{
var allVersions = packages
.Select(x => ConvertToVersion(x.Version.Version))
.Select(x => x.Version.Version.GetFromUmbracoString(reduceToConfigured: false))
.WhereNotNull()
.Distinct()
.OrderBy(x => x)
@@ -354,11 +357,11 @@ namespace OurUmbraco.Repository.Services
/// Order from latest package version and lastest min umb version and start from the top, we want to return the most recent
/// available package version for the current Umbraco version
/// </returns>
private IEnumerable<PackageVersionSupport> GetAllStrictSupportedPackageVersions(IEnumerable<WikiFile> packages)
internal static IEnumerable<PackageVersionSupport> GetAllStrictSupportedPackageVersions(IEnumerable<WikiFile> packages)
{
var allVersions = packages
.Where(x => x.MinimumVersionStrict.IsNullOrWhiteSpace() == false)
.Select(x => new PackageVersionSupport(x.Id, ConvertToVersion(x.Version.Version), ConvertToVersion(x.MinimumVersionStrict)))
.Select(x => new PackageVersionSupport(x.Id, x.Version.Version.GetFromUmbracoString(reduceToConfigured: false), x.MinimumVersionStrict.GetFromUmbracoString(reduceToConfigured: false)))
.Where(x => x.PackageVersion != null && x.MinUmbracoVersion != null)
.OrderByDescending(x => x.PackageVersion)
.ThenByDescending(x => x.MinUmbracoVersion)
@@ -378,47 +381,12 @@ namespace OurUmbraco.Repository.Services
{
var allVersions = packages
.Where(x => x.MinimumVersionStrict.IsNullOrWhiteSpace())
.Select(x => new PackageVersionSupport(x.Id, ConvertToVersion(x.Version.Version), null))
.Select(x => new PackageVersionSupport(x.Id, x.Version.Version.GetFromUmbracoString(reduceToConfigured:false), null))
.Where(x => x.PackageVersion != null)
.OrderByDescending(x => x.PackageVersion)
.ToArray();
return allVersions;
}
/// <summary>
/// Try to convert the string to a real version based on legacy version formats
/// </summary>
/// <param name="ver"></param>
/// <returns></returns>
private System.Version ConvertToVersion(string ver)
{
string normalized = ver;
if (ver.InvariantStartsWith("v"))
{
if (ver.InvariantStartsWith("v4"))
{
normalized = string.Concat(ver.Replace("v4", "4."), ".0");
}
else
{
normalized = string.Join(".", ver.ToCharArray().Skip(1));
}
}
if (normalized.EndsWith(".x"))
{
normalized = normalized.TrimEnd(".x").EnsureEndsWith(".0");
}
System.Version result;
if (System.Version.TryParse(normalized, out result))
{
return result;
}
return null;
}
}
private PackageOwnerInfo GetPackageOwnerInfo(int ownerId, bool openForCollab, int contentId)
{