人工智能生成文本检测在实践中使用有效性探讨

 token_perplexity = (probs.log() * -1.0).exp()
 print(f"Token Perplexity: {token_perplexity}")

 sentence_perplexity = (probs.log() * -1.0).mean().exp().item()
 print(f"Sentence Perplexity: {sentence_perplexity:.2f}")

 def get_pointwise_loss(self, inputs: List[str], tok):
     self.model.eval()
     all_probs = []
     with torch.inference_mode():
         for input in inputs:
             ids_list: List[int] = tok.encode(input).ids
             # ids has shape (1, len(ids_list))
             ids: Torch.Tensor = torch.tensor(ids_list, device=self.device).unsqueeze(0)
 
             # probs below is the probability that the token at that location
             # completes the sentence (in ids) so far.
             y = self.model(ids)
             criterion = nn.CrossEntropyLoss(reduction='none', ignore_index=0)
 
             # Compute the loss starting from the 2nd token in the model's output.
             loss = criterion(y[:,:,:-1], ids[:,1:])
 
             # To compute the probability of each token, we need to compute the
             # negative of the log loss and exponentiate it.
             loss = loss * -1.0
 
             # Set the probabilities that we are not interested in to -inf.
             # This is done to make the softmax set these values to 0.0
             loss[loss == 0.0] = float("-inf")
 
             # probs holds the probability of each token's prediction
             # starting from the 2nd token since we don't want to include
             # the probability of the model predicting the beginning of
             # a sentence given no existing sentence context.
             #
             # To compute perplexity, we should probably ignore the first
             # handful of predictions of the model since there's insufficient
             # context. We don’t do that here, though.
             probs = loss.exp()
             all_probs.append(probs)
         #
     #
     return all_probs
 #

 def get_html_for_token_perplexity(tok, sentence, tok_ppx, model_ppx):
     tokens = tok.encode(sentence).tokens
     ids = tok.encode(sentence).ids
     cleaned_tokens = []
     for word in tokens:
         m = list(map(ord, word))
         m = list(map(lambda x: x if x != 288 else ord(' '), m))
         m = list(map(chr, m))
         m = ''.join(m)
         cleaned_tokens.append(m)
     #
     html = [
         f"<span>{cleaned_tokens[0]}</span>",
     ]
     for ct, ppx in zip(cleaned_tokens[1:], tok_ppx):
         color = "black"
         if ppx.item() >= 0:
             if ppx.item() <= model_ppx * 1.1:
                 color = "red"
             else:
                 color = "green"
             #
         #
         html.append(f"<span style='color:{color};'>{ct}</span>")
     #
     return "".join(html)
 #

 import nltk
 from collections import Counter
 from sklearn.feature_extraction.text import CountVectorizer
 from textblob import TextBlob
 import numpy as np
 
 text = "This is some sample text to detect. This is a text written by a human? or is it?!"
 
 # 1. N-gram Analysis
 def ngram_analysis(text, n=2):
     n_grams = nltk.ngrams(text.split(), n)
     freq_dist = nltk.FreqDist(n_grams)
     print(freq_dist)
 
 ngram_analysis(text)
 
 # 2. Perplexity
 # Perplexity calculation typically requires a language model
 # This is just an example of how perplexity might be calculated given a probability distribution
 def perplexity(text, model=None):
     # This is a placeholder. In practice, we would use the model to get the probability of each word
     prob_dist = nltk.FreqDist(text.split())
     entropy = -1 * sum([p * np.log(p) for p in prob_dist.values()])
     return np.exp(entropy)
 
 print(perplexity(text))
 
 # 3. Burstiness
 def burstiness(text):
     word_counts = Counter(text.split())
     burstiness = len(word_counts) / np.std(list(word_counts.values()))
     return burstiness
 
 print(burstiness(text))
 
 # 4. Stylometry
 def stylometry(text):
     blob = TextBlob(text)
     avg_sentence_length = sum(len(sentence.words) for sentence in blob.sentences) / len(blob.sentences)
     passive_voice = text.lower().count('was') + text.lower().count('were')
     vocabulary_richness = len(set(text.split())) / len(text.split())
     return avg_sentence_length, passive_voice, vocabulary_richness
 
 print(stylometry(text))
 
 # 5. Consistency and Coherence Analysis
 # Again, this is a simple way to explain the idea of calculating the consistency of a text. In reality, more complex algorithms are used.
 def consistency(text):
     sentences = text.split(".")
     topics = [sentence.split()[0] for sentence in sentences if sentence]
     topic_changes = len(set(topics))
     return topic_changes
 
 print(consistency(text))