Merge pull request #3079 from abpframework/Docs-Update-label-improvements

Docs update label improvements

modules/docs/app/VoloDocs.EntityFrameworkCore/Migrations/20200312050853_init.Designer.cs

@@ -10,7 +10,7 @@ using VoloDocs.EntityFrameworkCore;
 namespace VoloDocs.EntityFrameworkCore.Migrations
 {
     [DbContext(typeof(VoloDocsDbContext))]
-    [Migration("20200218014727_init")]
+    [Migration("20200312050853_init")]
     partial class init
     {
         protected override void BuildTargetModel(ModelBuilder modelBuilder)
@@ -492,6 +492,9 @@ namespace VoloDocs.EntityFrameworkCore.Migrations
                     b.Property<DateTime>("LastCachedTime")
                         .HasColumnType("datetime2");
 
+                    b.Property<DateTime?>("LastSignificantUpdateTime")
+                        .HasColumnType("datetime2");
+
                     b.Property<DateTime>("LastUpdatedTime")
                         .HasColumnType("datetime2");
 

modules/docs/app/VoloDocs.EntityFrameworkCore/Migrations/20200312050853_init.cs

@@ -132,6 +132,7 @@ namespace VoloDocs.EntityFrameworkCore.Migrations
                     LocalDirectory = table.Column<string>(maxLength: 512, nullable: true),
                     CreationTime = table.Column<DateTime>(nullable: false),
                     LastUpdatedTime = table.Column<DateTime>(nullable: false),
+                    LastSignificantUpdateTime = table.Column<DateTime>(nullable: true),
                     LastCachedTime = table.Column<DateTime>(nullable: false)
                 },
                 constraints: table =>

modules/docs/app/VoloDocs.EntityFrameworkCore/Migrations/VoloDocsDbContextModelSnapshot.cs

@@ -490,6 +490,9 @@ namespace VoloDocs.EntityFrameworkCore.Migrations
                     b.Property<DateTime>("LastCachedTime")
                         .HasColumnType("datetime2");
 
+                    b.Property<DateTime?>("LastSignificantUpdateTime")
+                        .HasColumnType("datetime2");
+
                     b.Property<DateTime>("LastUpdatedTime")
                         .HasColumnType("datetime2");
 

modules/docs/src/Volo.Docs.Application/Volo/Docs/Documents/DocumentAppService.cs

@@ -96,6 +96,7 @@ namespace Volo.Docs.Documents
                 {
                     leaf.CreationTime = documentUpdateInfo.CreationTime;
                     leaf.LastUpdatedTime = documentUpdateInfo.LastUpdatedTime;
+                    leaf.LastSignificantUpdateTime = documentUpdateInfo.LastSignificantUpdateTime;
                 }
             }
 
@@ -189,12 +190,12 @@ namespace Volo.Docs.Documents
         {
             version = string.IsNullOrWhiteSpace(version) ? project.LatestVersionBranchName : version;
 
-            async Task<DocumentWithDetailsDto> GetDocumentAsync()
+            async Task<DocumentWithDetailsDto> GetDocumentAsync(Document oldDocument = null)
             {
                 Logger.LogInformation($"Not found in the cache. Requesting {documentName} from the source...");
 
                 var source = _documentStoreFactory.Create(project.DocumentStoreType);
-                var sourceDocument = await source.GetDocumentAsync(project, documentName, languageCode, version);
+                var sourceDocument = await source.GetDocumentAsync(project, documentName, languageCode, version, oldDocument?.LastSignificantUpdateTime);
 
                 await _documentRepository.DeleteAsync(project.Id, sourceDocument.Name, sourceDocument.LanguageCode, sourceDocument.Version);
                 await _documentRepository.InsertAsync(sourceDocument, true);
@@ -206,7 +207,8 @@ namespace Volo.Docs.Documents
                 {
                     Name = sourceDocument.Name,
                     CreationTime = sourceDocument.CreationTime,
-                    LastUpdatedTime = sourceDocument.LastUpdatedTime
+                    LastUpdatedTime = sourceDocument.LastUpdatedTime,
+                    LastSignificantUpdateTime = sourceDocument.LastSignificantUpdateTime
                 });
 
                 return CreateDocumentWithDetailsDto(project, sourceDocument);
@@ -229,7 +231,7 @@ namespace Volo.Docs.Documents
                 //TODO: Configurable cache time?
                 document.LastCachedTime + TimeSpan.FromHours(2) < DateTime.Now)
             {
-                return await GetDocumentAsync();
+                return await GetDocumentAsync(document);
             }
 
             var cacheKey = $"DocumentUpdateInfo{document.ProjectId}#{document.Name}#{document.LanguageCode}#{document.Version}";
@@ -238,6 +240,7 @@ namespace Volo.Docs.Documents
                 Name = document.Name,
                 CreationTime = document.CreationTime,
                 LastUpdatedTime = document.LastUpdatedTime,
+                LastSignificantUpdateTime = document.LastSignificantUpdateTime
             });
 
             return CreateDocumentWithDetailsDto(project, document);

modules/docs/src/Volo.Docs.Domain.Shared/Volo/Docs/Documents/DocumentUpdateInfo.cs

@@ -10,5 +10,7 @@ namespace Volo.Docs.Documents
         public virtual DateTime CreationTime { get; set; }
 
         public virtual DateTime LastUpdatedTime { get; set; }
+
+        public DateTime? LastSignificantUpdateTime { get; set; }
     }
 }

modules/docs/src/Volo.Docs.Domain.Shared/Volo/Docs/Documents/NavigationNode.cs

@@ -26,6 +26,8 @@ namespace Volo.Docs.Documents
 
         public virtual DateTime? LastUpdatedTime { get; set; }
 
+        public DateTime? LastSignificantUpdateTime { get; set; }
+
         public bool IsSelected(string documentName)
         {
             if (documentName == null)

modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/Document.cs

@@ -34,6 +34,8 @@ namespace Volo.Docs.Documents
         public virtual DateTime CreationTime { get; set; }
         
         public virtual DateTime LastUpdatedTime { get; set; }
+        
+        public virtual DateTime? LastSignificantUpdateTime { get; set; }
 
         public virtual DateTime LastCachedTime { get; set; }
 
@@ -60,7 +62,8 @@ namespace Volo.Docs.Documents
             [NotNull] string localDirectory,
             DateTime creationTime,
             DateTime lastUpdatedTime,
-            DateTime lastCachedTime
+            DateTime lastCachedTime,
+            DateTime? lastSignificantUpdateTime = null
         )
         {
             Id = id;
@@ -80,6 +83,7 @@ namespace Volo.Docs.Documents
             CreationTime = creationTime;
             LastUpdatedTime = lastUpdatedTime;
             LastCachedTime = lastCachedTime;
+            LastSignificantUpdateTime = lastSignificantUpdateTime;
 
             Contributors = new List<DocumentContributor>();
             ExtraProperties = new Dictionary<string, object>();

modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentSource.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Volo.Abp.Domain.Services;
@@ -8,7 +9,7 @@ namespace Volo.Docs.Documents
 {
     public interface IDocumentSource : IDomainService
     {
-        Task<Document> GetDocumentAsync(Project project, string documentName, string languageCode, string version);
+        Task<Document> GetDocumentAsync(Project project, string documentName, string languageCode, string version, DateTime? lastKnownSignificantUpdateTime = null);
 
         Task<List<VersionInfo>> GetVersionsAsync(Project project);
 

modules/docs/src/Volo.Docs.Domain/Volo/Docs/FileSystem/Documents/FileSystemDocumentSource.cs

@@ -16,7 +16,7 @@ namespace Volo.Docs.FileSystem.Documents
     {
         public const string Type = "FileSystem";
 
-        public async Task<Document> GetDocumentAsync(Project project, string documentName, string languageCode, string version)
+        public async Task<Document> GetDocumentAsync(Project project, string documentName, string languageCode, string version, DateTime? lastKnownSignificantUpdateTime = null)
         {
             var projectFolder = project.GetFileSystemPath();
             var path = Path.Combine(projectFolder, languageCode, documentName);

modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs

@@ -21,13 +21,15 @@ namespace Volo.Docs.GitHub.Documents
         public const string Type = "GitHub";
 
         private readonly IGithubRepositoryManager _githubRepositoryManager;
+        private readonly IGithubPatchAnalyzer _githubPatchAnalyzer;
 
-        public GithubDocumentSource(IGithubRepositoryManager githubRepositoryManager)
+        public GithubDocumentSource(IGithubRepositoryManager githubRepositoryManager, IGithubPatchAnalyzer githubPatchAnalyzer)
         {
             _githubRepositoryManager = githubRepositoryManager;
+            _githubPatchAnalyzer = githubPatchAnalyzer;
         }
-        
-        public virtual async Task<Document> GetDocumentAsync(Project project, string documentName, string languageCode, string version)
+
+        public virtual async Task<Document> GetDocumentAsync(Project project, string documentName, string languageCode, string version, DateTime? lastKnownSignificantUpdateTime = null)
         {
             var token = project.GetGitHubAccessTokenOrNull();
             var rootUrl = project.GetGitHubUrl(version);
@@ -46,25 +48,38 @@ namespace Volo.Docs.GitHub.Documents
                 fileName = documentName.Substring(documentName.LastIndexOf('/') + 1);
             }
 
-            var fileCommits = await GetFileCommitsAsync(project, version, $"docs/{languageCode}/{documentName}");
+            var fileCommits = await GetFileCommitsAsync(project, version, project.GetGitHubInnerUrl(languageCode, documentName));
+
+            var documentCreationTime = fileCommits.LastOrDefault()?.Commit.Author.Date.DateTime ?? DateTime.MinValue;
+
+            var lastSignificantUpdateTime = !isNavigationDocument && !isParameterDocument && version == project.LatestVersionBranchName ?
+                await GetLastSignificantUpdateTime(
+                    fileCommits,
+                    project, 
+                    project.GetGitHubInnerUrl(languageCode, documentName),
+                    lastKnownSignificantUpdateTime,
+                    documentCreationTime
+                    ) ?? lastKnownSignificantUpdateTime 
+                : null;
 
-            var document=  new Document(GuidGenerator.Create(), 
-                project.Id, 
-                documentName, 
-                version, 
+            var document = new Document(GuidGenerator.Create(),
+                project.Id,
+                documentName,
+                version,
                 languageCode,
-                fileName, 
+                fileName,
                 await DownloadWebContentAsStringAsync(rawDocumentUrl, token, userAgent),
-                project.Format, 
-                editLink, 
+                project.Format,
+                editLink,
                 rootUrl,
-                rawRootUrl, 
+                rawRootUrl,
                 localDirectory,
-                fileCommits.LastOrDefault()?.Commit.Author.Date.DateTime ?? DateTime.MinValue,
+                documentCreationTime,
                 fileCommits.FirstOrDefault()?.Commit.Author.Date.DateTime ?? DateTime.MinValue,
-                DateTime.Now);
+                DateTime.Now,
+                lastSignificantUpdateTime);
 
-            var authors =  fileCommits
+            var authors = fileCommits
                 .Where(x => x.Author != null)
                 .Select(x => x.Author)
                 .GroupBy(x => x.Id)
@@ -82,6 +97,42 @@ namespace Volo.Docs.GitHub.Documents
             return document;
         }
 
+        private async Task<DateTime?> GetLastSignificantUpdateTime(
+            IReadOnlyList<GitHubCommit> fileCommits,
+            Project project,
+            string fileName,
+            DateTime? lastKnownSignificantUpdateTime,
+            DateTime documentCreationTime)
+        {
+            if (!fileCommits.Any())
+            {
+                return null;
+            }
+
+            var fileCommitsAfterCreation = fileCommits.Take(fileCommits.Count - 1);
+
+            var commitsToEvaluate = (lastKnownSignificantUpdateTime != null
+                ? fileCommitsAfterCreation.Where(c => c.Commit.Author.Date.DateTime > lastKnownSignificantUpdateTime)
+                : fileCommitsAfterCreation).Where(c => c.Commit.Author.Date.DateTime > DateTime.Now.AddDays(-14));
+
+            foreach (var gitHubCommit in commitsToEvaluate)
+            {
+
+                var fullCommit = await _githubRepositoryManager.GetSingleCommitsAsync(
+                    GetOwnerNameFromUrl(project.GetGitHubUrl()),
+                    GetRepositoryNameFromUrl(project.GetGitHubUrl()),
+                    gitHubCommit.Sha,
+                    project.GetGitHubAccessTokenOrNull());
+
+                if (_githubPatchAnalyzer.HasPatchSignificantChanges(fullCommit.Files.First(f => f.Filename == fileName).Patch))
+                {
+                    return gitHubCommit.Commit.Author.Date.DateTime;
+                }
+            }
+
+            return null;
+        }
+
         public async Task<List<VersionInfo>> GetVersionsAsync(Project project)
         {
             List<VersionInfo> versions;

modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubPatchAnalyzer.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Volo.Abp.Domain.Services;
+
+namespace Volo.Docs.GitHub.Documents
+{
+    public class GithubPatchAnalyzer : DomainService, IGithubPatchAnalyzer
+    {
+        public bool HasPatchSignificantChanges(string patch)
+        {
+            var changes = GetChanges(patch);
+
+            return IsChangesSignificant(changes);
+        }
+
+
+        private bool IsChangesSignificant(CommitChanges change)
+        {
+            if (CompareLineCount(change))
+            {
+                return true;
+            }
+
+            if (CompareWordCount(change))
+            {
+                return true;
+            }
+
+            if (CompareWords(change))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        private static bool CompareLineCount(CommitChanges change)
+        {
+            return Math.Abs(change.NewLines.Count - change.OldLines.Count) >= 3;
+        }
+
+        private static bool CompareWordCount(CommitChanges change)
+        {
+            var wordCountInNewLines =
+                string.Join(" ", change.NewLines).Split(" ").Count(s => !string.IsNullOrWhiteSpace(s));
+            var wordCountInOldLines =
+                string.Join(" ", change.OldLines).Split(" ").Count(s => !string.IsNullOrWhiteSpace(s));
+
+            return Math.Abs(wordCountInNewLines - wordCountInOldLines) >= 15;
+        }
+
+        private bool CompareWords(CommitChanges change)
+        {
+            var wordsInNewLines = GetDistinctWordsFromLineList(change.NewLines);
+            var wordsInOldLines = GetDistinctWordsFromLineList(change.OldLines);
+
+            var differentWordsInNewLines = wordsInNewLines.Except(wordsInOldLines).Count();
+            var differentWordsInOldLines = wordsInOldLines.Except(wordsInNewLines).Count();
+
+            return differentWordsInNewLines + differentWordsInOldLines >= 10;
+        }
+
+        private CommitChanges GetChanges(string patch)
+        {
+            var changes = new List<CommitChanges>();
+            var pathSplited = patch.Split("@@");
+            var changeList = pathSplited.Where(s => !string.IsNullOrWhiteSpace(s)).Where((c, i) => i % 2 == 1).ToList();
+
+            foreach (var change in changeList)
+            {
+                var commitChange = new CommitChanges();
+                var lines = change.Split("\n");
+
+                commitChange.OldLines.AddRange(lines.Where(l => l.StartsWith("-")).Select(l => l.Substring(1)).Where(l => !string.IsNullOrWhiteSpace(l)));
+                commitChange.NewLines.AddRange(lines.Where(l => l.StartsWith("+")).Select(l => l.Substring(1)).Where(l => !string.IsNullOrWhiteSpace(l)));
+
+                changes.Add(commitChange);
+            }
+
+            return MergeChanges(changes);
+        }
+        private CommitChanges MergeChanges(List<CommitChanges> changes)
+        {
+            var mergedChanges = new CommitChanges();
+
+            foreach (var commitChanges in changes)
+            {
+                mergedChanges.NewLines.AddRange(commitChanges.NewLines);
+                mergedChanges.OldLines.AddRange(commitChanges.OldLines);
+            }
+
+            return mergedChanges;
+        }
+
+        private List<string> GetDistinctWordsFromLineList(List<string> lines)
+        {
+            return string.Join(" ", lines).Split(" ").Where(s => !string.IsNullOrWhiteSpace(s))
+                .Select(TrimAndRemovePunctuation).Distinct().ToList();
+        }
+
+        private string TrimAndRemovePunctuation(string str)
+        {
+            return new string(str.Trim().ToCharArray().Where(c => !char.IsPunctuation(c)).ToArray());
+        }
+
+        private class CommitChanges
+        {
+            public List<string> OldLines { get; set; }
+
+            public List<string> NewLines { get; set; }
+
+            public CommitChanges()
+            {
+                OldLines = new List<string>();
+                NewLines = new List<string>();
+            }
+        }
+    }
+}

modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubRepositoryManager.cs

@@ -75,5 +75,15 @@ namespace Volo.Docs.GitHub.Documents
             var request = new CommitRequest { Path = filename, Sha = version };
             return await client.Repository.Commit.GetAll(repo.Id, request);
         }
+
+        public async Task<GitHubCommit> GetSingleCommitsAsync(string name, string repositoryName, string sha, string token)
+        {
+            var client = token.IsNullOrWhiteSpace()
+                ? new GitHubClient(new ProductHeaderValue(name))
+                : new GitHubClient(new ProductHeaderValue(name), new InMemoryCredentialStore(new Credentials(token)));
+
+            var repo = await client.Repository.Get(name, repositoryName);
+            return await client.Repository.Commit.Get(repo.Id, sha);
+        }
     }
 }

modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/IGithubPatchAnalyzer.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Volo.Abp.Domain.Services;
+
+namespace Volo.Docs.GitHub.Documents
+{
+    public interface IGithubPatchAnalyzer : IDomainService
+    {
+        bool HasPatchSignificantChanges(string patch);
+    }
+}

modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/IGithubRepositoryManager.cs

@@ -16,5 +16,7 @@ namespace Volo.Docs.GitHub.Documents
         Task<IReadOnlyList<Release>> GetReleasesAsync(string name, string repositoryName, string token);
 
         Task<IReadOnlyList<GitHubCommit>> GetFileCommitsAsync(string name, string repositoryName, string version, string filename, string token);
+
+        Task<GitHubCommit> GetSingleCommitsAsync(string name, string repositoryName, string sha, string token);
     }
 }

modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Projects/ProjectGithubExtensions.cs

@@ -14,6 +14,12 @@ namespace Volo.Docs.GitHub.Projects
             return project.ExtraProperties["GitHubRootUrl"] as string;
         }
 
+        public static string GetGitHubInnerUrl([NotNull] this Project project, string languageCode, string documentName)
+        {
+            return project
+                       .GetGitHubUrl().Split("{version}")[1].EnsureEndsWith('/').TrimStart('/') + languageCode + '/' + documentName;
+        }
+
         public static string GetGitHubUrl([NotNull] this Project project, string version)
         {
             return project

modules/docs/src/Volo.Docs.Web/Areas/Documents/TagHelpers/TreeTagHelper.cs

@@ -127,16 +127,16 @@ namespace Volo.Docs.Areas.Documents.TagHelpers
 
                 if (!node.Path.IsNullOrWhiteSpace() && node.CreationTime.HasValue && node.LastUpdatedTime.HasValue)
                 {
-                    var newBadge = "<span class='badge badge-primary ml-2' title=\"" + _localizer["NewExplanation"] + "\">" + _localizer["New"] + "</span>";
-                    var updBadge = "<span class='badge badge-light ml-2' title=\"" + _localizer["UpdatedExplanation"] + "\">" + _localizer["Upd"] + "</span>";
-
                     if(node.CreationTime + TimeSpan.FromDays(14) > DateTime.Now)
                     {
-                        badge = newBadge;
+                        var newBadge = "<span class='badge badge-primary ml-2' title=\"" + _localizer["NewExplanation"] + "\">" + _localizer["New"] + "</span>";
+                        badge += newBadge;
                     }
-                    else if (node.LastUpdatedTime + TimeSpan.FromDays(14) > DateTime.Now)
+                    
+                    if (node.LastSignificantUpdateTime != null && node.LastSignificantUpdateTime + TimeSpan.FromDays(14) > DateTime.Now)
                     {
-                        badge = updBadge;
+                        var updBadge = "<span class='badge badge-light ml-2' title=\"" + _localizer["UpdatedExplanation"] + "\">" + _localizer["Upd"] + "</span>";
+                        badge += updBadge;
                     }
                 }