from sklearn.model_selection import train_test_split

train_data, test_data = train_test_split(data, test_size=0.3, random_state=42)
train_data, val_data = train_test_split(train_data, test_size=0.2, random_state=42)

from nltk.corpus import wordnet

def synonym_replacement(text):
    words = text.split()
    new_words = words.copy()
    for i in range(len(words)):
        synonyms = wordnet.synsets(words[i])
        if synonyms:
            new_words[i] = synonyms[0].lemmas()[0].name()
    return ' '.join(new_words)

from googletrans import Translator

def back_translation(text, src='en', dest='de'):
    translator = Translator()
    translated = translator.translate(text, src=src, dest=dest).text
    back_translated = translator.translate(translated, src=dest, dest=src).text
    return back_translated

import random

def random_deletion(text, p=0.1):
    words = text.split()
    if len(words) == 1:
        return text
    new_words = [word for word in words if random.uniform(0, 1) > p]
    return ' '.join(new_words)

from wordcloud import WordCloud
import matplotlib.pyplot as plt

def plot_wordcloud(text):
    wordcloud = WordCloud(width=800, height=400).generate(text)
    plt.figure(figsize=(10, 5))
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.show()

import pandas as pd
import deepl

# DeepL API认证密钥
auth_key = ""
translator = deepl.Translator(auth_key)

# 读取XLSX文件
data = pd.read_csv('ds.xlsx')

# 定义一个标志位，用于指示是否停止翻译
stop_translation = False

# 翻译文本的函数，从英文翻译成中文
def translate_to_chinese(text):
    global stop_translation
    if stop_translation:
        return text  # 如果已经出现异常，直接返回原文
    try:
        translation = translator.translate_text(text, target_lang="ZH")
        return translation.text
    except Exception as e:
        print(f"翻译文本时出错: {e}")
        stop_translation = True  # 发生异常，设置标志位，停止后续翻译
        return text

# 翻译 'question' 列
data['question'] = data['question'].apply(translate_to_chinese)

# 翻译 'answer' 列
data['answer'] = data['answer'].apply(translate_to_chinese)

# 显示翻译后的数据的前几行
print(data.head())

# 将翻译后的数据保存到一个CSV文件
data.to_csv('translated_ds.csv', index=False)

import pandas as pd
import deepl

# DeepL API认证密钥
auth_key = "cAjDPhRVq1V2y80O"
translator = deepl.Translator(auth_key)

# 读取CSV文件
data = pd.read_csv('java.csv')

# 定义一个标志位，用于指示是否停止翻译
stop_translation = False


# 翻译文本的函数，从英文翻译成中文
def translate_to_chinese(text):
    global stop_translation
    if stop_translation:
        return text  # 如果已经出现异常，直接返回原文
    try:
        translation = translator.translate_text(text, target_lang="ZH")
        return translation.text
    except Exception as e:
        print(f"翻译文本时出错: {e}")
        stop_translation = True  # 发生异常，设置标志位，停止后续翻译
        return text


# 翻译 'question' 列
data['question'] = data['question'].apply(translate_to_chinese)

# 翻译 'answer' 列
data['answer'] = data['answer'].apply(translate_to_chinese)

# 显示翻译后的数据的前几行
# print(data.head())

# 将翻译后的数据保存到一个CSV文件
data.to_csv('translated_java.csv', index=False)

import pandas as pd
import deepl

auth_key = "cAjDPhRVq1V2y80O"
translator = deepl.Translator(auth_key)

data = pd.read_parquet('os.parquet')

# 定义一个标志位，用于指示是否停止翻译
stop_translation = False


# 翻译文本的函数，从英文翻译成中文
def translate_to_chinese(text):
    global stop_translation
    if stop_translation:
        return text  # 如果已经出现异常，直接返回原文
    try:
        translation = translator.translate_text(text, target_lang="ZH")
        return translation.text
    except Exception as e:
        print(f"翻译文本时出错: {e}")
        stop_translation = True  # 发生异常，设置标志位，停止后续翻译
        return text


# 翻译 'question' 列
data['question'] = data['question'].apply(translate_to_chinese)

# 翻译 'answer' 列
data['answer'] = data['answer'].apply(translate_to_chinese)

# 显示翻译后的数据的前几行
# print(data.head())

# 将翻译后的数据保存到一个CSV文件
data.to_csv('translated_os.csv', index=False)

import pandas as pd
import deepl

# DeepL API认证密钥
auth_key = "cAjDPhRVq1V2y80O"
translator = deepl.Translator(auth_key)

# 读取CSV文件
data = pd.read_csv('os.csv')

# 定义一个标志位，用于指示是否停止翻译
stop_translation = False

# 翻译文本的函数，从俄语翻译成中文
def translate_to_chinese(text):
    global stop_translation
    if stop_translation:
        return text  # 如果已经出现异常，直接返回原文
    try:
        translation = translator.translate_text(text, source_lang="RU", target_lang="ZH")
        return translation.text
    except Exception as e:
        print(f"翻译文本时出错: {e}")
        stop_translation = True  # 发生异常，设置标志位，停止后续翻译
        return text

# 翻译 'question' 列
data['question'] = data['question'].apply(translate_to_chinese)

# 翻译 'answer' 列
data['answer'] = data['answer'].apply(translate_to_chinese)

# 显示翻译后的数据的前几行
print(data.head())

# 将翻译后的数据保存到一个CSV文件
data.to_csv('translated_os.csv', index=False)

import pandas as pd

# 读取CSV文件
data = pd.read_csv('data.csv')

# 过滤数据，保留满足条件的行
filtered_data = data[(data['question'].str.len() >= 5) & (data['question'].str.len() <= 50) &
                     (data['answer'].str.len() >= 50) & (data['answer'].str.len() <= 512)]

# 保存过滤后的数据到新的CSV文件
filtered_data.to_csv('filtered_data.csv', index=False)

import pandas as pd
from difflib import SequenceMatcher

def similar(a, b):
    return SequenceMatcher(None, a, b).ratio()

# 读取CSV文件
data = pd.read_csv('data.csv')

# 初始化要删除的索引列表
indices_to_drop = set()

# 检查问题列的相似度
for i in range(len(data) - 1):
    for j in range(i + 1, len(data)):
        # 比较问题列
        if similar(data.loc[i, 'question'], data.loc[j, 'question']) > 0.8:
            indices_to_drop.add(j)  # 添加问题二的索引到删除列表

        # 比较答案列
        if similar(data.loc[i, 'answer'], data.loc[j, 'answer']) > 0.8:
            indices_to_drop.add(j)  # 添加答案二的索引到删除列表

# 删除相似度高的行
data = data.drop(indices_to_drop)

# 保存过滤后的数据到新的CSV文件
data.to_csv('filtered_data.csv', index=False)

// hash data of corpus to deduplication (read and save)
     var hashFile = PropertiesReader.get("dedeplication_hash_path");

     //1.rule
     var ruleQuality = new RuleQuality();

     //2.sensitivity and advertising detection
     var simpleSenDetectionProcessor = SimpleSenDetectionProcessor.newInstance();
     var senDetection = simpleSenDetectionProcessor.getKWSeeker("sensitive_words_path");

     var ad_detect_model_path = PropertiesReader.get("ad_detect_model_path");
     var ad_dict_path = PropertiesReader.get("ad_dict_path");
     var stop_words_path = PropertiesReader.get("stop_words_path");
     var adDetection = new AdDetection(ad_detect_model_path, ad_dict_path, stop_words_path);

     //3.text deduplication
     var deDuplication = new DeDuplication(4, 3);

     //4.quality evaluation
     var ngramModelPath = PropertiesReader.get("language_model_path");
     var qualityEvaluation = new QualityEvaluation(ngramModelPath);

     // load hash
     if(Files.exists(Paths.get(hashFile))) {
         deDuplication.loadHash(hashFile);
     }

     var corpusQuality = new CorpusQuality(ruleQuality, senDetection, adDetection, deDuplication, qualityEvaluation, 100);
     var corpus = "对未按土地、环保和投资管理等法律法规履行相关手续或手续不符合规定的违规项目，地方政府要按照要求进行全面清理。一，凡是未开工的违规项目，一律不得开工建设；二，凡是不符合产业政策、准入标准、环保要求的违规项目一律停建。";
     var result = corpusQuality.quality(corpus);
     System.out.println(result);

     // save hash
     deDuplication.saveHash(hashFile);

import pandas as pd
import json

# 读取CSV文件
data = pd.read_csv('cleaned_translated_train.csv')

# 转换数据为JSON格式，每一行变成一个字典
json_data = [
    {
        "instruction": row["question"],
        "input": "",
        "output": row["answer"]
    }
    for index, row in data.iterrows()
]

# 将JSON数据保存到文件
with open('data_for_training.json', 'w', encoding='utf-8') as json_file:
    json.dump(json_data, json_file, ensure_ascii=False, indent=4)

import json
import os

# 所有JSON文件都存放在一个名为 'json_files' 的目录中
directory_path = 'json_files'
all_data = []

# 遍历目录中的每个文件
for filename in os.listdir(directory_path):
    if filename.endswith('.json'):
        file_path = os.path.join(directory_path, filename)
        # 打开并读取JSON文件
        with open(file_path, 'r', encoding='utf-8') as file:
            data = json.load(file)
            all_data.append(data)  # 将读取的数据添加到列表中

# 将合并后的数据保存到一个新的JSON文件中
with open('combined_data.json', 'w', encoding='utf-8') as output_file:
    json.dump(all_data, output_file, ensure_ascii=False, indent=4)