科大讯飞 中文成语填空挑战赛baseline

from sklearn.metrics import accuracy_score
y_pred = [0, 2, 1, 3]
y_true = [0, 1, 2, 3]
accuracy_score(y_true, y_pred)

import re
import pandas as pd
from tqdm import tqdm

train = pd.read_csv('data/train.csv', sep='\t')
test = pd.read_csv('data/test.csv', sep='\t')

print(train)
print(test)


def process_text(text):
    return re.sub(' +', ' ', text).strip()


def get_question(text):
    """
    根据[MASK][MASK][MASK][MASK]获取问题
    :param text:
    :return:
    """
    sentences = re.split('(。|！|\!|\.|？|\?)', text)  # 保留分割符
    for sent in sentences:
        if '[MASK][MASK][MASK][MASK]' in sent:
            return sent
    return text


cols = [
    "Unnamed: 0",
    "video-id",
    "fold-ind",  # q_id
    "startphrase",
    "sent1",  # content
    "sent2",  # question
    "gold-source",
    "ending0", "ending1", "ending2", "ending3",  # choice
    "label"]

# ======================================================
# 生成训练集
# ======================================================
res = []

for idx, row in tqdm(train.iterrows()):
    q_id = f'train_{idx}'
    content = row['text']
    content = process_text(content)
    question = get_question(content)
    modified_choices = eval(row['candidate'])
    label = modified_choices.index(row['label'])
    ## Hard-code for swag format!
    res.append(("",
                "",
                q_id,
                "",
                content,
                question,
                "",
                modified_choices[0],
                modified_choices[1],
                modified_choices[2],
                modified_choices[3],
                label))
df = pd.DataFrame(res, columns=cols)

@dataclass
class DataCollatorForMultipleChoice:
    """
    Data collator that will dynamically pad the inputs for multiple choice received.
    Args:
        tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
            The tokenizer used for encoding the data.
        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
            among:
            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
              sequence if provided).
            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
              maximum acceptable input length for the model if that argument is not provided.
            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
              different lengths).
        max_length (:obj:`int`, `optional`):
            Maximum length of the returned list and optionally padding length (see above).
        pad_to_multiple_of (:obj:`int`, `optional`):
            If set will pad the sequence to a multiple of the provided value.
            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
            7.5 (Volta).
    """

    tokenizer: PreTrainedTokenizerBase
    padding: Union[bool, str, PaddingStrategy] = True
    max_length: Optional[int] = None
    pad_to_multiple_of: Optional[int] = None

    def __call__(self, features):
        label_name = "label" if "label" in features[0].keys() else "labels"
        labels = [feature.pop(label_name) for feature in features]
        batch_size = len(features)
        num_choices = len(features[0]["input_ids"])
        flattened_features = [
            [{k: v[i] for k, v in feature.items()} for i in range(num_choices)] for feature in features
        ]
        flattened_features = sum(flattened_features, [])

        batch = self.tokenizer.pad(
            flattened_features,
            padding=self.padding,
            max_length=self.max_length,
            pad_to_multiple_of=self.pad_to_multiple_of,
            return_tensors="pt",
        )

        # Un-flatten
        batch = {k: v.view(batch_size, num_choices, -1) for k, v in batch.items()}
        # Add back labels
        batch["labels"] = torch.tensor(labels, dtype=torch.int64)
        return batch

 # Metric
    def compute_metrics(eval_predictions):
        predictions, label_ids = eval_predictions
        preds = np.argmax(predictions, axis=1)
        return {"accuracy": (preds == label_ids).astype(np.float32).mean().item()}

    # Initialize our Trainer
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=tokenized_datasets["train"] if training_args.do_train else None,
        eval_dataset=tokenized_datasets["validation"] if training_args.do_eval else None,
        tokenizer=tokenizer,
        data_collator=data_collator,
        compute_metrics=compute_metrics,
    )

    # Training
    if training_args.do_train:
        if last_checkpoint is not None:
            checkpoint = last_checkpoint
        elif os.path.isdir(model_args.model_name_or_path):
            checkpoint = model_args.model_name_or_path
        else:
            checkpoint = None
        train_result = trainer.train(resume_from_checkpoint=checkpoint)
        trainer.save_model()  # Saves the tokenizer too for easy upload

        output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
        if trainer.is_world_process_zero():
            with open(output_train_file, "w") as writer:
                logger.info("***** Train results *****")
                for key, value in sorted(train_result.metrics.items()):
                    logger.info(f"  {key} = {value}")
                    writer.write(f"{key} = {value}\n")

            # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
            trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))

#!/bin/bash

python -u baseline.py \
  --model_name_or_path 'hfl/chinese-xlnet-base' \
  --do_train \
  --do_eval \
  --do_predict \
  --logging_steps=100 \
  --max_seq_length 200 \
  --train_file data/new_train.csv \
  --validation_file data/new_valid.csv \
  --test_file data/new_test.csv \
  --learning_rate 3e-5 \
  --num_train_epochs 2 \
  --output_dir 'models/xlnet' \
  --gradient_accumulation_steps 4 \
  --per_device_eval_batch_size 16 \
  --per_device_train_batch_size 16 \
  --overwrite_output