Added docstrings to the code and made JuliaHealthLLM, a package

Author: ParamThakkar123

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.5"
 manifest_format = "2.0"
-project_hash = "7466aa371ae0c2e72224798b8708eaac8f0c47a6"
+project_hash = "714c8f58ac5b36d35301b506df3812f7dc600a22"
 
 [[deps.AliasTables]]
 deps = ["PtrArrays", "Random"]

Project.toml

@@ -6,9 +6,6 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DotEnv = "4dc1fcf4-5e3b-5448-94ab-0c38ec0385c1"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 DrWatson = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
-HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 TextAnalysis = "a2db99b7-8b79-58f8-94bf-bbc811eef33d"
 

scripts/clone.jl

@@ -1,28 +1,48 @@
-using LibGit2
-using Downloads
+"""
+    clone_repositories(repos::Vector{String}, data_dir::String)
 
+Clones a list of Git repositories into the specified directory.
+
+# Arguments
+- `repos::Vector{String}`: A vector of repository URLs to be cloned.
+- `data_dir::String`: The directory where the repositories will be cloned.
+
+# Behavior
+- If the target directory does not exist, it will be created.
+- If a repository has already been cloned (i.e., its directory exists), it will be skipped.
+- If cloning fails for any repository, an error message will be printed.
+
+# Example
+```julia
 repos = [
-    "https://github.com/JuliaHealth/MedEval3D.jl.git",
+    "https://github.com/JuliaLang/julia.git",
+    "https://github.com/JuliaHealth/MedEval3D.jl.git"
 ]
-
 data_dir = "data/exp_raw"
+clone_repositories(repos, data_dir)
+```
 
-if !isdir(data_dir)
-    mkdir(data_dir)
-end
+"""
+function clone_repositories(repos::Vector{String}, data_dir::String)
+    # Ensure the target directory exists
+    if !isdir(data_dir)
+        mkdir(data_dir)
+    end
 
-for repo in repos
-    repo_name = split(repo, "/")[end]
-    repo_path = joinpath(data_dir, replace(repo_name, ".git" => ""))
-    
-    if isdir(repo_path)
-        println("Skipping $repo, already cloned.")
-    else
-        try
-            LibGit2.clone(repo, repo_path)
-            println("Successfully cloned $repo")
-        catch e
-            println("Failed to clone $repo: $e")
+    # Iterate through the list of repositories and clone them
+    for repo in repos
+        repo_name = split(repo, "/")[end]
+        repo_path = joinpath(data_dir, replace(repo_name, ".git" => ""))
+        
+        if isdir(repo_path)
+            println("Skipping $repo, already cloned.")
+        else
+            try
+                LibGit2.clone(repo, repo_path)
+                println("Successfully cloned $repo")
+            catch e
+                println("Failed to clone $repo: $e")
+            end
         end
     end
 end

scripts/knowledge.jl

@@ -1,7 +1,29 @@
-using LibGit2
-using TextAnalysis
-using Serialization
+"""
+    normalize_repo_url(input::String) -> String
 
+Normalizes a repository URL or name into a full GitHub URL.
+
+# Arguments
+- `input::String`: The repository name (e.g., "owner/repo") or a full Git URL.
+
+# Returns
+- A normalized GitHub URL ending with `.git`.
+
+# Throws
+- An error if the input is not a valid repository name or URL.
+
+# Behavior
+- If the input is already a valid Git URL (e.g., "https://github.com/owner/repo.git"), it is returned as-is.
+- If the input is a repository name in the format "owner/repo", it is converted to a full GitHub URL.
+- If the input is invalid, an error is raised.
+
+# Example
+```julia
+normalize_repo_url("JuliaLang/julia") # "https://github.com/JuliaLang/julia.git"
+normalize_repo_url("https://github.com/JuliaLang/julia") # "https://github.com/JuliaLang/julia.git"
+normalize_repo_url("invalid_repo") # Error: Invalid repository URL or name
+```
+"""
 function normalize_repo_url(input::String)
     if occursin(r"^https?://", input)
         return endswith(input, ".git") ? input : input * ".git"
@@ -12,18 +34,119 @@ function normalize_repo_url(input::String)
     end
 end
 
+"""
+    clone_repo(repo_url::String, local_path::String)
+
+Clones a Git repository to a specified local path.
+
+# Arguments
+- `repo_url::String`: The URL of the repository to clone.
+- `local_path::String`: The local directory where the repository will be cloned.
+
+# Behavior
+- If the cloning is successful, the repository is cloned to the specified path.
+- Prints progress messages during the cloning process.
+
+# Throws
+- An error if the cloning process fails (e.g., invalid URL, network issues, or permission errors).
+
+# Example
+```julia
+clone_repo("https://github.com/JuliaLang/julia.git", "local_julia_repo")
+```
+"""
 function clone_repo(repo_url::String, local_path::String)
     println("Cloning repository from $repo_url to $local_path...")
     LibGit2.clone(repo_url, local_path)
     println("Repository cloned successfully.")
 end
 
+"""
+    is_textual_file(filename::String) -> Bool
+
+Checks if a file is a textual file based on its extension.
+
+# Arguments
+- `filename::String`: The name of the file to check.
+
+# Returns
+- `true` if the file has a textual extension (e.g., `.md`, `.jl`, `.txt`, etc.), otherwise `false`.
+
+# Behavior
+- The function uses a regular expression to match common textual file extensions.
+- The check is case-insensitive.
+
+# Example
+```julia
+is_textual_file("README.md") # true
+is_textual_file("script.jl") # true
+is_textual_file("image.png") # false
+```
+"""
 function is_textual_file(filename::String)
     textual_extensions = r"\.(md|jl|txt|rst|doc|docx|py|cpp|c|h|java|sh)$"
     return occursin(textual_extensions, lowercase(filename))
 end
 
-function build_corpus_from_repo(repo_input::String)
+"""
+    process_directory(dir_path::String, documents::Vector{AbstractDocument})
+
+Recursively processes a directory to extract textual files and add them to a document collection.
+
+# Arguments
+- `dir_path::String`: The directory to process.
+- `documents::Vector{AbstractDocument}`: A vector to store the extracted documents.
+
+# Behavior
+- Textual files are identified using the `is_textual_file` function.
+- The content of each textual file is read and converted into a `StringDocument` object, which is added to the `documents` vector.
+- Subdirectories are explored recursively, skipping hidden directories (those starting with a dot).
+
+# Example
+```julia
+documents = AbstractDocument[]
+process_directory("path/to/repo", documents)
+println("Number of documents processed: ", length(documents))
+```
+"""
+function process_directory(dir_path::String, documents::Vector{AbstractDocument})
+    for entry in readdir(dir_path, join=true)
+        if isfile(entry) && is_textual_file(entry)
+            println("Processing file: $entry")
+            content = read(entry, String)
+            doc = StringDocument(content)
+            push!(documents, doc)
+        elseif isdir(entry) && !occursin(r"^\.", basename(entry))
+            println("Exploring directory: $entry...")
+            process_directory(entry, documents)  # Recursive call
+        end
+    end
+end
+
+"""
+    build_corpus_from_repo(repo_input::String) -> Corpus
+
+Clones a repository (if not already cloned), processes its files, and builds a corpus.
+
+# Arguments
+- `repo_input::String`: The repository name (e.g., "owner/repo") or a full Git URL.
+
+# Returns
+- A `Corpus` object containing the textual content of the repository.
+
+# Behavior
+- The repository URL is normalized using `normalize_repo_url`.
+- If the repository is already cloned, it reuses the local copy; otherwise, it clones the repository using `clone_repo`.
+- The repository's directory is processed recursively using `process_directory` to extract textual files.
+- A `Corpus` object is created from the extracted documents, and its lexicon and index are updated.
+
+# Example
+```julia
+corpus = build_corpus_from_repo("JuliaLang/julia")
+println("Corpus contains ", length(corpus), " documents.")
+```
+"""
+function build_corpus_from_repo(repo_input::String)::Corpus
     repo_url = normalize_repo_url(repo_input)
     repo_name = split(repo_url, "/")[end] |> x -> replace(x, ".git" => "")
     local_path = joinpath(pwd(), "repo_$repo_name")
@@ -35,22 +158,7 @@ function build_corpus_from_repo(repo_input::String)
     end
 
     documents = AbstractDocument[]
-    
-    function process_directory(dir_path)
-        for entry in readdir(dir_path, join=true)
-            if isfile(entry) && is_textual_file(entry)
-                println("Processing file: $entry")
-                content = read(entry, String)
-                doc = StringDocument(content)
-                push!(documents, doc)
-            elseif isdir(entry) && !occursin(r"^\.", basename(entry))
-                println("Exploring directory: $entry...")
-                process_directory(entry)
-            end
-        end
-    end
-    
-    process_directory(local_path)
+    process_directory(local_path, documents)  # Call the standalone function
 
     corpus = Corpus(documents)
     println("Updating corpus lexicon and index...")
@@ -60,50 +168,54 @@ function build_corpus_from_repo(repo_input::String)
     return corpus
 end
 
+"""
+    save_corpus(corpus::Corpus, repo_input::String)
+
+Saves a corpus to a file in the current working directory.
+
+# Arguments
+- `corpus::Corpus`: The corpus to save.
+- `repo_input::String`: The repository name or URL (used to generate the filename).
+
+# Behavior
+- The repository name is normalized using `normalize_repo_url` to create a safe filename.
+- The corpus is serialized and saved to a `.jls` file in the current working directory.
+- Ensures the target directory exists before saving the file.
+- Prints progress messages during the saving process.
+
+# Throws
+- An error message is printed if the saving process fails (e.g., permission issues or invalid paths).
+
+# Example
+```julia
+corpus = build_corpus_from_repo("JuliaLang/julia")
+save_corpus(corpus, "JuliaLang/julia")
+```
+"""
 function save_corpus(corpus::Corpus, repo_input::String)
     current_dir = pwd()
-    safe_repo_name = split(normalize_repo_url(repo_input), "/")[end] |> x -> replace(x, ".git" => "") |> x -> replace(x, r"[^a-zA-Z0-9_]" => "_")
+    safe_repo_name = split(normalize_repo_url(repo_input), "/")[end] |> 
+                     x -> replace(x, ".git" => "") |> 
+                     x -> replace(x, r"[^a-zA-Z0-9_]" => "_")
     filename = joinpath(current_dir, "corpus_$(safe_repo_name).jls")
 
-    println("Saving corpus to $filename...")
-    open(filename, "w") do io
-        serialize(io, corpus)
-    end
-    println("Corpus saved successfully.")
-end
-
-function main()
-    repo_input = ""
-    
-    if isinteractive()
-        println("Please enter the GitHub repository URL or name (e.g., 'https://github.com/JuliaLang/julia.git' or 'JuliaLang/julia'):")
-        repo_input = readline()
-    else
-        if length(ARGS) > 0
-            repo_input = ARGS[1]
-            println("Using repository from command-line argument: $repo_input")
-        else
-            repo_input = "https://github.com/JuliaLang/julia.git"
-            println("No repository specified. Using default: $repo_input")
-        end
-    end
-    
-    if isempty(strip(repo_input))
-        println("Error: Repository input cannot be empty. Using default 'https://github.com/JuliaLang/julia.git'.")
-        repo_input = "https://github.com/JuliaLang/julia.git"
-    else
-        println("Using repository: $repo_input")
-    end
-    
     try
-        corpus = build_corpus_from_repo(repo_input)
-        save_corpus(corpus, repo_input)
-        return corpus
+        println("Saving corpus to $filename...")
+        
+        # Ensure the directory exists
+        dir_path = dirname(filename)
+        if !isdir(dir_path)
+            println("Directory $dir_path does not exist. Creating it...")
+            mkdir(dir_path)
+        end
+        
+        # Save the corpus to the file
+        open(filename, "w") do io
+            serialize(io, corpus)
+        end
+        
+        println("Corpus saved successfully to $filename.")
     catch e
-        println("Error occurred: $e")
-        println("Please check the repository URL or name and ensure it is valid.")
-        return nothing
+        println("Failed to save corpus: $e")
     end
-end
-
-corpus = main()
+end