swcs/NetUnicodeInfo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added support for emoji properties.

Browse Source
This commit is contained in:
GoldenCrystal
2017-06-25 17:14:03 +02:00
parent f9976c7a03
commit 642b675b3d
15 changed files with 185 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
UnicodeInformation.Builder/HttpDataSource.cs
+4 -15
View File
@@ -1,28 +1,17 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.IO;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
namespace System.Unicode.Builder
{
public class HttpDataSource : IDataSource
{
public static readonly Uri UnicodeCharacterDataUri = new Uri("http://www.unicode.org/Public/UCD/latest/ucd/", UriKind.Absolute);
private readonly HttpClient httpClient;
private readonly Uri baseUri;
public HttpDataSource()
: this(UnicodeCharacterDataUri)
public HttpDataSource(Uri baseUri, HttpClient httpClient)
{
}
public HttpDataSource(Uri baseUri)
{
this.httpClient = new HttpClient();
this.httpClient = httpClient ?? new HttpClient();
this.baseUri = baseUri;
}
UnicodeInformation.Builder/Program.cs
+28 -22
View File
@@ -11,6 +11,9 @@ namespace System.Unicode.Builder
{
internal class Program
{
public static readonly Uri UnicodeCharacterDataUri = new Uri("http://www.unicode.org/Public/UCD/latest/ucd/", UriKind.Absolute);
public static readonly Uri EmojiDataUri = new Uri("http://www.unicode.org/Public/emoji/latest/", UriKind.Absolute);
public const string UnihanDirectoryName = "Unihan";
public const string UnihanArchiveName = "Unihan.zip";
public const string UcdDirectoryName = "UCD";
@@ -36,22 +39,28 @@ namespace System.Unicode.Builder
"Unihan_IRGSources.txt",
};
public static readonly string[] emojiRequiredFiles = new[]
{
"emoji-data.txt",
"emoji-sequences.txt",
"emoji-variation-sequences.txt",
"emoji-zwj-sequences.txt",
};
private static HttpClient httpClient;
// The only purpose of this is for tests…
internal static void SetHttpMessageHandler(HttpMessageHandler handler)
{
httpClient = new HttpClient(handler ?? new HttpClientHandler());
}
private static HttpClient HttpClient { get { return httpClient ?? (httpClient = new HttpClient()); } }
private static byte[] DownloadDataArchive(string archiveName)
{
return HttpClient.GetByteArrayAsync(HttpDataSource.UnicodeCharacterDataUri + archiveName).Result;
}
internal static IDataSource GetDataSource(string archiveName, string directoryName, string[] requiredFiles, bool? shouldDownload, bool? shouldSaveArchive, bool? shouldExtract)
internal static HttpClient HttpClient { get { return httpClient ?? (httpClient = new HttpClient()); } }
private static Task<byte[]> DownloadDataArchiveAsync(string archiveName)
=> HttpClient.GetByteArrayAsync(UnicodeCharacterDataUri + archiveName);
internal static async Task<IDataSource> GetDataSourceAsync(string archiveName, string directoryName, string[] requiredFiles, bool? shouldDownload, bool? shouldSaveArchive, bool? shouldExtract)
{
string baseDirectory = Directory.GetCurrentDirectory();
string dataDirectory = Path.Combine(baseDirectory, UcdDirectoryName);
@@ -94,15 +103,14 @@ namespace System.Unicode.Builder
if (shouldDownload != false)
{
var dataArchiveData = DownloadDataArchive(archiveName);
var dataArchiveData = await DownloadDataArchiveAsync(archiveName).ConfigureAwait(false);
if (shouldSaveArchive == true)
{
var stream = File.Open(dataArchiveFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
try
using (var stream = File.Open(dataArchiveFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read))
{
stream.Write(dataArchiveData, 0, dataArchiveData.Length);
dataArchiveData = null; // Release the reference now, since we won't need it anymore.
await stream.WriteAsync(dataArchiveData, 0, dataArchiveData.Length).ConfigureAwait(false);
dataArchiveData = null; // Release the reference now, since we won't need it anymore.
if (shouldExtract == true)
{
@@ -118,11 +126,6 @@ namespace System.Unicode.Builder
return new ZipDataSource(stream);
}
}
catch
{
stream.Dispose();
throw;
}
}
else
{
@@ -133,18 +136,21 @@ namespace System.Unicode.Builder
throw new InvalidOperationException();
}
private static void Main(string[] args)
private static async Task MainAsync(string[] args)
{
UnicodeInfoBuilder data;
using (var ucdSource = GetDataSource(UcdArchiveName, UcdDirectoryName, ucdRequiredFiles, null, null, null))
using (var unihanSource = GetDataSource(UnihanArchiveName, UnihanDirectoryName, unihanRequiredFiles, null, null, null))
using (var ucdSource = await GetDataSourceAsync(UcdArchiveName, UcdDirectoryName, ucdRequiredFiles, null, null, null))
using (var unihanSource = await GetDataSourceAsync(UnihanArchiveName, UnihanDirectoryName, unihanRequiredFiles, null, null, null))
using (var emojiSource = new HttpDataSource(EmojiDataUri, HttpClient))
{
data = UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource).Result;
data = await UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource, emojiSource);
}
using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false))
data.WriteToStream(stream);
}
private static void Main(string[] args) => MainAsync(args).Wait();
}
}
UnicodeInformation.Builder/UnicodeDataProcessor.cs
+18 -1
View File
@@ -23,6 +23,7 @@ namespace System.Unicode.Builder
public const string UnihanVariantsFileName = "Unihan_Variants.txt";
public const string UnihanNumericValuesFileName = "Unihan_NumericValues.txt";
public const string UnihanIrgSourcesFileName = "Unihan_IRGSources.txt";
public const string EmojiDataFileName = "emoji-data.txt";
private static string ParseSimpleCaseMapping(string mapping)
{
@@ -36,13 +37,14 @@ namespace System.Unicode.Builder
return string.IsNullOrEmpty(s) ? null : s;
}
public static async Task<UnicodeInfoBuilder> BuildDataAsync(IDataSource ucdSource, IDataSource unihanSource)
public static async Task<UnicodeInfoBuilder> BuildDataAsync(IDataSource ucdSource, IDataSource unihanSource, IDataSource emojiSource)
{
var builder = new UnicodeInfoBuilder(await ReadUnicodeVersionAsync(ucdSource).ConfigureAwait(false));
await ProcessUnicodeDataFile(ucdSource, builder).ConfigureAwait(false);
await ProcessPropListFile(ucdSource, builder).ConfigureAwait(false);
await ProcessDerivedCorePropertiesFile(ucdSource, builder).ConfigureAwait(false);
await ProcessEmojiDataFile(emojiSource, builder).ConfigureAwait(false);
await ProcessCjkRadicalsFile(ucdSource, builder).ConfigureAwait(false);
await ProcessNameAliasesFile(ucdSource, builder).ConfigureAwait(false);
await ProcessNamesListFile(ucdSource, builder).ConfigureAwait(false);
@@ -219,6 +221,21 @@ namespace System.Unicode.Builder
}
}
private static async Task ProcessEmojiDataFile(IDataSource emojiSource, UnicodeInfoBuilder builder)
{
using (var reader = new UnicodeDataFileReader(await emojiSource.OpenDataFileAsync(EmojiDataFileName).ConfigureAwait(false), ';'))
{
while (reader.MoveToNextLine())
{
var range = UnicodeCodePointRange.Parse(reader.ReadTrimmedField());
if (EnumHelper<EmojiProperties>.TryGetNamedValue(reader.ReadTrimmedField(), out var property))
{
builder.SetProperties(property, range);
}
}
}
}
private static async Task ProcessCjkRadicalsFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
{
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(CjkRadicalsFileName).ConfigureAwait(false), ';'))
UnicodeInformation.Builder/UnicodeInfoBuilder.cs
+32 -1
View File
@@ -226,7 +226,7 @@ namespace System.Unicode.Builder
|| ucdEntries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|| ucdEntries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
{
throw new InvalidOperationException();
throw new InvalidOperationException("Unable to find code point for setting core property.");
}
int i = firstIndex;
@@ -241,6 +241,37 @@ namespace System.Unicode.Builder
}
}
public void SetProperties(EmojiProperties property, UnicodeCodePointRange codePointRange)
{
int firstIndex = FindUcdCodePoint(codePointRange.FirstCodePoint);
int lastIndex = FindUcdCodePoint(codePointRange.LastCodePoint);
if (firstIndex < 0 && lastIndex < 0)
{
Insert(new UnicodeCharacterDataBuilder(codePointRange) { EmojiProperties = property });
return;
}
if (firstIndex < 0
|| lastIndex < 0
|| ucdEntries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|| ucdEntries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
{
throw new InvalidOperationException("Unable to find code point for setting emoji property.");
}
int i = firstIndex;
while (true)
{
ucdEntries[i].EmojiProperties |= property;
if (i == lastIndex) break;
++i;
}
}
public void SetRadicalInfo(int radicalIndex, CjkRadicalData data)
{
if (radicalIndex < 1 || radicalIndex > CjkRadicalCount) throw new ArgumentOutOfRangeException(nameof(radicalIndex));
UnicodeInformation.Builder/ucd.dat
BIN
View File
Binary file not shown.