From 02380443cd3fe8b60de488ed6e843382855645cd Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Fri, 20 Dec 2024 16:05:27 -0700 Subject: [PATCH 1/6] Updated BaseRepository to include default branch, adding default branch in gitlab, setting repo to be reindexed if default branch changes --- packages/backend/src/gitlab.ts | 1 + packages/backend/src/main.ts | 3 ++- packages/backend/src/types.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index b586f023..bfec6c81 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -114,6 +114,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon cloneUrl: cloneUrl.toString(), path: repoPath, isStale: false, + defaultBranch: project.default_branch, isFork, isArchived: project.archived, topics: project.topics ?? [], diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index c5aa2b66..bca0de34 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -154,7 +154,8 @@ export const isRepoReindexingRequired = (previous: Repository, current: Reposito return ( !arraysEqualShallow(previous.branches, current.branches) || - !arraysEqualShallow(previous.tags, current.tags) + !arraysEqualShallow(previous.tags, current.tags) || + previous.defaultBranch !== current.defaultBranch ); } diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 2b0eca3a..2aefb3be 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -10,6 +10,7 @@ interface BaseRepository { codeHost?: string; topics?: string[]; sizeInBytes?: number; + defaultBranch: string; } export interface GitRepository extends BaseRepository { From 3e1d92af3caf95200e9d6ebdb4cf6e02ed6246b7 Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Thu, 26 Dec 2024 14:25:57 -0700 Subject: [PATCH 2/6] GitHub and Gitea include default branch info --- .idea/inspectionProfiles/Project_Default.xml | 6 + .idea/misc.xml | 6 + .idea/modules.xml | 8 + .idea/sourcebot.iml | 9 + .idea/vcs.xml | 6 + .idea/workspace.xml | 202 +++++++++++++++++++ TODO.txt | 2 + packages/backend/src/gitea.ts | 1 + packages/backend/src/github.ts | 4 +- packages/backend/src/types.ts | 2 +- 10 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/sourcebot.iml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 TODO.txt diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000..03d9549e --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..cc6eae03 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..5a28fb59 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/sourcebot.iml b/.idea/sourcebot.iml new file mode 100644 index 00000000..d6ebd480 --- /dev/null +++ b/.idea/sourcebot.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 00000000..f758998d --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,202 @@ + + + + + + + + + + + + + + { + "associatedIndex": 7 +} + + + + index.ts.executor": "Debug", + "Node.js.backend > main.ts.executor": "Debug", + "Node.js.index.ts.executor": "Debug", + "Node.js.main.ts.executor": "Debug", + "RunOnceActivity.ShowReadmeOnStart": "true", + "Vitest.backend > utils.test.ts.executor": "Debug", + "Vitest.should filter repos by micromatch pattern.executor": "Debug", + "Vitest.should filter repos by name exact match.executor": "Debug", + "Vitest.should work.executor": "Run", + "git-widget-placeholder": "index-default-branch", + "kotlin-language-version-configured": "true", + "node.js.detected.package.eslint": "true", + "node.js.detected.package.tslint": "true", + "node.js.selected.package.eslint": "(autodetect)", + "node.js.selected.package.tslint": "(autodetect)", + "nodejs_package_manager_path": "yarn", + "npm.dev.executor": "Debug", + "project.structure.last.edited": "Project", + "project.structure.proportion": "0.0", + "project.structure.side.proportion": "0.2", + "settings.editor.selected.configurable": "preferences.editor", + "ts.external.directory.path": "/Users/snagie/Documents/sourcebot/node_modules/typescript/lib", + "vue.rearranger.settings.migration": "true" + }, + "keyToStringList": { + "com.intellij.ide.scratch.ScratchImplUtil$2/New Scratch File": [ + "JSON" + ] + } +}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1731521287325 + + + + + + + + + + file://$PROJECT_DIR$/packages/backend/src/gitlab.ts + 110 + + + file://$PROJECT_DIR$/packages/backend/src/gitlab.ts + 114 + + + file://$PROJECT_DIR$/packages/backend/src/utils.ts + 38 + + + file://$PROJECT_DIR$/packages/backend/src/gitlab.ts + 24 + + + + file://$PROJECT_DIR$/packages/backend/src/gitlab.ts + 96 + + + + file://$PROJECT_DIR$/packages/backend/src/gitlab.ts + 102 + + + + + + \ No newline at end of file diff --git a/TODO.txt b/TODO.txt new file mode 100644 index 00000000..42ba6807 --- /dev/null +++ b/TODO.txt @@ -0,0 +1,2 @@ +1. Verify that reindexing via the "isRepoReindexingRequired" method does what you think it does +2. Match other git services to gitlab (i.e. make sure github, gittea, local, and gerrit all include the default branch in there) diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index 831d38a6..2d58003c 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -52,6 +52,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte id: repoId, cloneUrl: cloneUrl.toString(), path: repoPath, + defaultBranch: repo.default_branch, isStale: false, isFork: repo.fork!, isArchived: !!repo.archived, diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index 15f4b05a..3373d967 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -21,8 +21,9 @@ type OctokitRepository = { subscribers_count?: number, forks_count?: number, archived?: boolean, + default_branch?: string, topics?: string[], - size?: number, + size?: number } export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: AbortSignal, ctx: AppContext) => { @@ -79,6 +80,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo id: repoId, cloneUrl: cloneUrl.toString(), path: repoPath, + defaultBranch: repo.default_branch, isStale: false, isFork: repo.fork, isArchived: !!repo.archived, diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 2aefb3be..e6b91d90 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -10,7 +10,7 @@ interface BaseRepository { codeHost?: string; topics?: string[]; sizeInBytes?: number; - defaultBranch: string; + defaultBranch?: string; } export interface GitRepository extends BaseRepository { From 383582c05e6bd4a1549beb0fc1dc4d892b5ea333 Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Thu, 26 Dec 2024 14:45:43 -0700 Subject: [PATCH 3/6] Zoekt indexes default branch --- packages/backend/src/zoekt.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index ffeadf7e..2f44dd5d 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -9,6 +9,7 @@ export const indexGitRepository = async (repo: GitRepository, settings: Settings ...repo.branches ?? [], ...repo.tags ?? [], ]; + if (repo.defaultBranch) revisions.push(repo.defaultBranch); const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} ${repo.path}`; From 8d0bd835dd611d0260ac51cbb16b3cf92cc3c2b5 Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Thu, 26 Dec 2024 14:50:23 -0700 Subject: [PATCH 4/6] Deleted editor files --- .idea/inspectionProfiles/Project_Default.xml | 6 - .idea/misc.xml | 6 - .idea/modules.xml | 8 - .idea/sourcebot.iml | 9 - .idea/vcs.xml | 6 - .idea/workspace.xml | 202 ------------------- TODO.txt | 2 - 7 files changed, 239 deletions(-) delete mode 100644 .idea/inspectionProfiles/Project_Default.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/sourcebot.iml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml delete mode 100644 TODO.txt diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 03d9549e..00000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index cc6eae03..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 5a28fb59..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/sourcebot.iml b/.idea/sourcebot.iml deleted file mode 100644 index d6ebd480..00000000 --- a/.idea/sourcebot.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddf..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index f758998d..00000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,202 +0,0 @@ - - - - - - - - - - - - - - { - "associatedIndex": 7 -} - - - - index.ts.executor": "Debug", - "Node.js.backend > main.ts.executor": "Debug", - "Node.js.index.ts.executor": "Debug", - "Node.js.main.ts.executor": "Debug", - "RunOnceActivity.ShowReadmeOnStart": "true", - "Vitest.backend > utils.test.ts.executor": "Debug", - "Vitest.should filter repos by micromatch pattern.executor": "Debug", - "Vitest.should filter repos by name exact match.executor": "Debug", - "Vitest.should work.executor": "Run", - "git-widget-placeholder": "index-default-branch", - "kotlin-language-version-configured": "true", - "node.js.detected.package.eslint": "true", - "node.js.detected.package.tslint": "true", - "node.js.selected.package.eslint": "(autodetect)", - "node.js.selected.package.tslint": "(autodetect)", - "nodejs_package_manager_path": "yarn", - "npm.dev.executor": "Debug", - "project.structure.last.edited": "Project", - "project.structure.proportion": "0.0", - "project.structure.side.proportion": "0.2", - "settings.editor.selected.configurable": "preferences.editor", - "ts.external.directory.path": "/Users/snagie/Documents/sourcebot/node_modules/typescript/lib", - "vue.rearranger.settings.migration": "true" - }, - "keyToStringList": { - "com.intellij.ide.scratch.ScratchImplUtil$2/New Scratch File": [ - "JSON" - ] - } -}]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1731521287325 - - - - - - - - - - file://$PROJECT_DIR$/packages/backend/src/gitlab.ts - 110 - - - file://$PROJECT_DIR$/packages/backend/src/gitlab.ts - 114 - - - file://$PROJECT_DIR$/packages/backend/src/utils.ts - 38 - - - file://$PROJECT_DIR$/packages/backend/src/gitlab.ts - 24 - - - - file://$PROJECT_DIR$/packages/backend/src/gitlab.ts - 96 - - - - file://$PROJECT_DIR$/packages/backend/src/gitlab.ts - 102 - - - - - - \ No newline at end of file diff --git a/TODO.txt b/TODO.txt deleted file mode 100644 index 42ba6807..00000000 --- a/TODO.txt +++ /dev/null @@ -1,2 +0,0 @@ -1. Verify that reindexing via the "isRepoReindexingRequired" method does what you think it does -2. Match other git services to gitlab (i.e. make sure github, gittea, local, and gerrit all include the default branch in there) From 92fbf51e897ff71ed857098b9acc6c08e5a5a2d7 Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Thu, 26 Dec 2024 15:24:28 -0700 Subject: [PATCH 5/6] Moved defaultBranch property to GitRepository instead of BaseRepository --- packages/backend/src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index e6b91d90..999f1ee0 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -10,7 +10,6 @@ interface BaseRepository { codeHost?: string; topics?: string[]; sizeInBytes?: number; - defaultBranch?: string; } export interface GitRepository extends BaseRepository { @@ -18,6 +17,7 @@ export interface GitRepository extends BaseRepository { cloneUrl: string; branches: string[]; tags: string[]; + defaultBranch?: string; gitConfigMetadata?: Record; } From 17cc16dbbb3591d4d0ff84dcc6300a218e3529b0 Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Fri, 27 Dec 2024 10:47:19 -0700 Subject: [PATCH 6/6] Updated defaultBranch comparison to account better for undefined and null values in isRepoReindexingRequired --- packages/backend/src/main.ts | 4 ++-- packages/backend/src/utils.ts | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index bca0de34..914009d2 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -9,7 +9,7 @@ import { AppContext, LocalRepository, GitRepository, Repository, Settings } from import { cloneRepository, fetchRepository } from "./git.js"; import { createLogger } from "./logger.js"; import { createRepository, Database, loadDB, updateRepository, updateSettings } from './db.js'; -import { arraysEqualShallow, isRemotePath, measure } from "./utils.js"; +import { arraysEqualShallow, stringsEqualFalseySafe, isRemotePath, measure } from "./utils.js"; import { DEFAULT_SETTINGS } from "./constants.js"; import stripJsonComments from 'strip-json-comments'; import { indexGitRepository, indexLocalRepository } from "./zoekt.js"; @@ -155,7 +155,7 @@ export const isRepoReindexingRequired = (previous: Repository, current: Reposito return ( !arraysEqualShallow(previous.branches, current.branches) || !arraysEqualShallow(previous.tags, current.tags) || - previous.defaultBranch !== current.defaultBranch + !stringsEqualFalseySafe(previous.defaultBranch, current.defaultBranch) ); } diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 7e94905a..677bb2e0 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -129,3 +129,9 @@ export const arraysEqualShallow = (a?: readonly T[], b?: readonly T[]) => { return true; } + +export const stringsEqualFalseySafe = (a?: string, b?: string): boolean => { + if (a === b) return true; + else if (!a && !b) return true; + else return false; +}