构建你自己的 AI 辅助编码助手：从 IDE 插件、代码数据生成和模型微调（万字长文）

<｜fim▁begin｜>def quick_sort(arr):
if len(arr) <= 1:
return arr
    pivot = arr[0]
    left = []
    right = []
<｜fim▁hole｜>
if arr[i] < pivot:
            left.append(arr[i])
else:
            right.append(arr[i])
return quick_sort(left) + [pivot] + quick_sort(right)<｜fim▁end｜>

fun createBlog(blogDto: CreateBlogDto): BlogPost{

}

val blogPost = BlogPost(
    title = blogDto.title,
    content = blogDto.content,
    author = blogDto.author
)
return blogRepository.save(blogPost)

fun createBlog(blogDto: CreateBlogDto): BlogPost{
//...
}

fun updateBlog(id: Long, blogDto: CreateBlogDto): BlogPost{
//...
}

fun deleteBlog(id: Long) {
//...
}

{
"prefix": "# Path: codeviz\\app.py\n#....",
"suffix": "if __name__ == '__main__':\r\n    app.run(debug=True)",
"isFimEnabled": true,
"promptElementRanges": [
{
"kind": "PathMarker",
"start": 0,
"end": 23
},
{
"kind": "SimilarFile",
"start": 23,
"end": 2219
},
{
"kind": "BeforeCursor",
"start": 2219,
"end": 3142
}
]
}

// Path: src/main/cc/unitmesh/demo/infrastructure/repositories/ProductRepository.java
// Compare this snippet from src/main/cc/unitmesh/demo/domain/product/Product.java:
// ....
// Compare this snippet from src/main/cc/unitmesh/demo/application/ProductService.java:
// ...
// @Component
// public class ProductService {
//     //...
// }
//
package cc.unitmesh.demo.repositories;

// ...

@Component
publicclassProductRepository{
//...

// here are related classes:
// 'filePath: /Users/phodal/IdeaProjects/untitled/src/main/java/cc/unitmesh/untitled/demo/service/BlogService.java
// class BlogService {
//   blogRepository
//   + public BlogPost createBlog(BlogPost blogDto)
//   + public BlogPost getBlogById(Long id)
//   + public BlogPost updateBlog(Long id, BlogPost blogDto)
//   + public void deleteBlog(Long id)
// }
// 'filePath: /Users/phodal/IdeaProjects/untitled/src/main/java/cc/unitmesh/untitled/demo/dto/CreateBlogRequest.java
// class CreateBlogRequest ...
// 'filePath: /Users/phodal/IdeaProjects/untitled/src/main/java/cc/unitmesh/untitled/demo/entity/BlogPost.java
// class BlogPost {...
@ApiOperation(value = "Create a new blog")
@PostMapping("/")
publicBlogPost createBlog(@RequestBodyCreateBlogRequest request) {

pluginSinceBuild = 223
pluginUntilBuild = 233.*

classAutoDevEditorListener: EditorFactoryListener{
override fun editorCreated(event: EditorFactoryEvent) {
//...
        editor.document.addDocumentListener(AutoDevDocumentListener(editor), editorDisposable)
        editor.caretModel.addCaretListener(AutoDevCaretListener(editor), editorDisposable)
//...
}

classAutoDevCaretListener(val editor: Editor) : CaretListener{
override fun caretPositionChanged(event: CaretEvent) {
//...
            val wasTypeOver = TypeOverHandler.getPendingTypeOverAndReset(editor)
//...
            llmInlayManager.disposeInlays(editor, InlayDisposeContext.CaretChange)
}
}

classAutoDevDocumentListener(val editor: Editor) : BulkAwareDocumentListener{
override fun documentChangedNonBulk(event: DocumentEvent) {
//...
            val llmInlayManager = LLMInlayManager.getInstance()
            llmInlayManager
.editorModified(editor, changeOffset)
}
}
}

<groupid="AutoDevIntentionsActionGroup"class="cc.unitmesh.devti.intentions.IntentionsActionGroup"
icon="cc.unitmesh.devti.AutoDevIcons.AI_COPILOT"searchable="false">
<add-to-groupgroup-id="ShowIntentionsGroup"relative-to-action="ShowIntentionActions"anchor="after"/>
</group>

java/   # Java 语言插件
  src/main/java/cc/unitmesh/autodev/ # Java 语言入口
  src/main/resources/META-INF/plugin.xml
plugin/ # 多平台入口
  src/main/resources/META-INF/plugin.xml
src/    # 即核心模块
  main/resource/META-INF/core.plugin.xml

<idea-pluginpackage="cc.unitmesh"xmlns:xi="http://www.w3.org/2001/XInclude"allow-bundled-update="true">
<xi:includehref="/META-INF/core.xml"xpointer="xpointer(/idea-plugin/*)"/>
<content>
<modulename="cc.unitmesh.java"/>
<!--  其它模块 -->
</content>
</idea-plugin>

<idea-pluginpackage="cc.unitmesh.java">
<!--suppress PluginXmlValidity -->
<dependencies>
<pluginid="com.intellij.modules.java"/>
<pluginid="org.jetbrains.plugins.gradle"/>
</dependencies>
</idea-plugin>

cc.unitmesh.javascript.xml
cc.unitmesh.rust.xml
cc.unitmesh.python.xml
cc.unitmesh.kotlin.xml
cc.unitmesh.java.xml
cc.unitmesh.go.xml
cc.unitmesh.cpp.xml

private fun findRelatedCode(container: CodeContainer): List<CodeDataStruct> {
// 1. collects all similar data structure by imports if exists in a file tree
    val byImports = container.Imports
.mapNotNull {
            context.fileTree[it.Source]?.container?.DataStructures
}
.flatten()

// 2. collects by inheritance tree for some node in the same package
    val byInheritance = container.DataStructures
.map {
(it.Implements+ it.Extend).mapNotNull { i ->
                context.fileTree[i]?.container?.DataStructures
}.flatten()
}
.flatten()

    val related = (byImports + byInheritance).distinctBy { it.NodeName}
// 3. convert all similar data structure to uml
return related
}

classRelatedCodeStrategyBuilder(private val context: JobContext) : CodeStrategyBuilder{
override fun build(): List<TypedIns> {
// ...
        val findRelatedCodeDs = findRelatedCode(container)
        val relatedCodePath = findRelatedCodeDs.map { it.FilePath}
        val jaccardSimilarity = SimilarChunker.pathLevelJaccardSimilarity(relatedCodePath, currentPath)
        val relatedCode = jaccardSimilarity.mapIndexed { index, d ->
            findRelatedCodeDs[index] to d
}.sortedByDescending {
            it.second
}.take(3).map {
            it.first
}
//...
}
}

fun pathLevelJaccardSimilarity(chunks: List<String>, text: String): List<Double> {
//...
}
fun tokenize(chunk: String): List<String> {
return chunk.split(Regex("[^a-zA-Z0-9]")).filter { it.isNotBlank() }
}
fun similarityScore(set1: Set<String>, set2: Set<String>): Double{
//...
}

;; methods
(method_declaration
  name: (identifier) @hoist.definition.method)

pub static JAVA: TSLanguageConfig= TSLanguageConfig{
    language_ids: &["Java"],
    file_extensions: &["java"],
    grammar: tree_sitter_java::language,
    scope_query: MemoizedQuery::new(include_str!("./scopes.scm")),
    hoverable_query: MemoizedQuery::new(
        r#"
[(identifier)
(type_identifier)] @hoverable
"#,
    ),
    namespaces: &[&[
        // variables
        "local",
        // functions
        "method",
        // namespacing, modules
        "package",
        "module",
        // types
        "class",
        "enum",
        "enumConstant",
        "record",
        "interface",
        "typedef",
        // misc.
        "label",
    ]],
};

from tree_sitter importTree

def chunker(
    tree: Tree,
    source_code: bytes,
    MAX_CHARS=512* 3,
    coalesce=50# Any chunk less than 50 characters long gets coalesced with the next chunk
) -> list[Span]:

# 1. Recursively form chunks based on the last post (https://docs.sweep.dev/blogs/chunking-2m-files)
def chunk_node(node: Node) -> list[Span]:
        chunks: list[Span] = []
        current_chunk: Span= Span(node.start_byte, node.start_byte)
        node_children = node.children
for child in node_children:
if child.end_byte - child.start_byte > MAX_CHARS:
                chunks.append(current_chunk)
                current_chunk = Span(child.end_byte, child.end_byte)
                chunks.extend(chunk_node(child))
elif child.end_byte - child.start_byte + len(current_chunk) > MAX_CHARS:
                chunks.append(current_chunk)
                current_chunk = Span(child.start_byte, child.end_byte)
else:
                current_chunk += Span(child.start_byte, child.end_byte)
        chunks.append(current_chunk)
return chunks
    chunks = chunk_node(tree.root_node)

# 2. Filling in the gaps
for prev, curr in zip(chunks[:-1], chunks[1:]):
        prev.end = curr.start
    curr.start = tree.root_node.end_byte

# 3. Combining small chunks with bigger ones
    new_chunks = []
    current_chunk = Span(0, 0)
for chunk in chunks:
        current_chunk += chunk
if non_whitespace_len(current_chunk.extract(source_code)) > coalesce \
and"\n"in current_chunk.extract(source_code):
            new_chunks.append(current_chunk)
            current_chunk = Span(chunk.end, chunk.end)
if len(current_chunk) > 0:
        new_chunks.append(current_chunk)

# 4. Changing line numbers
    line_chunks = [Span(get_line_number(chunk.start, source_code),
                    get_line_number(chunk.end, source_code)) for chunk in new_chunks]

# 5. Eliminating empty chunks
    line_chunks = [chunk for chunk in line_chunks if len(chunk) > 0]

return line_chunks

pip install -r requirements.txt
python server-python38.py

if __name__ == "__main__":
try:
        meta = requests.get('http://localhost:21999/gear-status', timeout=5).json()
        url = meta['links'].get('auxiliary')
if url:
print("打开该链接访问:", url)
exceptException:
pass

    uvicorn.run(app, host="0.0.0.0", port=8080)

{
"instruction": "Write unit test for following code.\n<SomeCode>",
"output": "<TestCode>"
}

{
"instruction": "Write unit test for following code.",
"input": "<SomeCode>",
"output": "<TestCode>"
}

DATA_PATH="/openbayes/home/summary.jsonl"
OUTPUT_PATH="/openbayes/home/output"

MODEL_PATH="/openbayes/input/input0/"

!cd DeepSeek-Coder/finetune && deepspeed finetune_deepseekcoder.py \
--model_name_or_path $MODEL_PATH \
--data_path $DATA_PATH \
--output_dir $OUTPUT_PATH \
--num_train_epochs 1 \
--model_max_length 1024 \
--per_device_train_batch_size 8 \
--per_device_eval_batch_size 1 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 375 \
--save_total_limit 10 \
--learning_rate 1e-4 \
--warmup_steps 10 \
--logging_steps 1 \
--lr_scheduler_type "cosine" \
--gradient_checkpointing True \
--report_to "tensorboard" \
--deepspeed configs/ds_config_zero3.json \
--bf16 True

`use_cache=True` is incompatible with gradient checkpointing. Setting`use_cache=False`...
0%|                                                   | 0/375[00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting`use_cache=False`...
{'loss': 0.6934, 'learning_rate': 0.0, 'epoch': 0.0}                            
{'loss': 0.3086, 'learning_rate': 3.0102999566398115e-05, 'epoch': 0.01}        
{'loss': 0.3693, 'learning_rate': 4.771212547196624e-05, 'epoch': 0.01}         
{'loss': 0.3374, 'learning_rate': 6.020599913279623e-05, 'epoch': 0.01}         
{'loss': 0.4744, 'learning_rate': 6.989700043360187e-05, 'epoch': 0.01}         
{'loss': 0.3465, 'learning_rate': 7.781512503836436e-05, 'epoch': 0.02}         
{'loss': 0.4258, 'learning_rate': 8.450980400142567e-05, 'epoch': 0.02}         
{'loss': 0.4027, 'learning_rate': 9.030899869919434e-05, 'epoch': 0.02}         
{'loss': 0.2844, 'learning_rate': 9.542425094393248e-05, 'epoch': 0.02}         
{'loss': 0.3783, 'learning_rate': 9.999999999999999e-05, 'epoch': 0.03}

Write unit test for following code.

${context.testFramework}
${context.coreFramework}
${context.testSpec}

```${context.language}
${context.related_model}
${context.selection}
```

val codeDir = GitUtil
.checkoutCode(config.url, config.branch, tempGitDir, config.gitDepth)
.toFile().canonicalFile

logger.info("start walk $codeDir")

val languageWorker = LanguageWorker()
val workerManager = WorkerManager(
WorkerContext(
        config.codeContextStrategies,
        config.codeQualityTypes,
        config.insOutputConfig,
        pureDataFileName = config.pureDataFileName(),
        config.completionTypes,
        config.maxCompletionEachFile,
        config.completionTypeSize,
        qualityThreshold = InsQualityThreshold(
            complexity = InsQualityThreshold.MAX_COMPLEXITY,
            fileSize = InsQualityThreshold.MAX_FILE_SIZE,
            maxLineInCode = config.maxLineInCode,
            maxCharInCode = config.maxCharInCode,
            maxTokenLength = config.maxTokenLength,
)
)
)
workerManager.init(codeDir, config.language)

fun filterByThreshold(job: InstructionFileJob) {
    val summary = job.fileSummary
if(!supportedExtensions.contains(summary.extension)) {
return
}

// limit by complexity
if(summary.complexity > context.qualityThreshold.complexity) {
        logger.info("skip file ${summary.location} for complexity ${summary.complexity}")
return
}

// like js minified file
if(summary.binary || summary.generated || summary.minified) {
return
}

// if the file size is too large, we just try 64k
if(summary.bytes > context.qualityThreshold.fileSize) {
        logger.info("skip file ${summary.location} for size ${summary.bytes}")
return
}

// limit by token length
    val encoded = enc.encode(job.code)
    val length = encoded.size
if(length > context.qualityThreshold.maxTokenLength) {
        logger.info("skip file ${summary.location} for over ${context.qualityThreshold.maxTokenLength} tokens")
        println("| filename: ${summary.filename} |  tokens: $length | complexity: ${summary.complexity} | code: ${summary.lines} | size: ${summary.bytes} | location: ${summary.location} |")
return
}

    val language = SupportedLang.from(summary.language)
    val worker = workers[language] ?: return
    worker.addJob(job)
}

val lists = jobs.map { job ->
    val jobContext = JobContext(
        job,
        context.qualityTypes,
        fileTree,
        context.insOutputConfig,
        context.completionTypes,
        context.maxCompletionInOneFile,
        project = ProjectContext(
            compositionDependency = context.compositionDependency,
),
        context.qualityThreshold
)

    context.codeContextStrategies.map { type ->
        val codeStrategyBuilder = type.builder(jobContext)
        codeStrategyBuilder.build()
}.flatten()
}.flatten()

fun create(types: List<CodeQualityType>, thresholds: Map<String, Int> = mapOf()): List<QualityAnalyser> {
return types.map { type ->
when(type) {
CodeQualityType.BadSmell-> BadsmellAnalyser(thresholds)
CodeQualityType.TestBadSmell-> TestBadsmellAnalyser(thresholds)
CodeQualityType.JavaController-> JavaControllerAnalyser(thresholds)
CodeQualityType.JavaRepository-> JavaRepositoryAnalyser(thresholds)
CodeQualityType.JavaService-> JavaServiceAnalyser(thresholds)
}
}
}