blob: bf2ae0bf9f48e795db67becc6f23d1ec4a92d22d (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include <iostream>
#include <iomanip>
#include <map>
#include <string>
#include "fsa.h"
#include "permuter.h"
#include "ngram.h"
#include "base64.h"
using namespace fsa;
int main(int argc, char **argv)
{
const unsigned int MAXQUERY = 10;
const unsigned int MAXGRAM = 6;
Permuter p;
NGram freq_s,gram,sorted_gram;
unsigned int freq;
Selector s(10);
std::string gstr;
if(argc!=3){
std::cerr << "usage: " << argv[0] << " plain_count_fsa_file sorted_count_fsa_file" << std::endl;
return 1;
}
FSA plain_fsa(argv[1]);
FSA sorted_fsa(argv[2]);
FSA::State state1(plain_fsa),state2(sorted_fsa);
while(!std::cin.eof()){
getline(std::cin,gstr);
gram.set(gstr);
if(gram.length()>1){
sorted_gram.set(gram);
sorted_gram.sort();
sorted_gram.uniq();
state1.startWord(gram[0]);
for(unsigned int i=1;state1.isValid()&&i<gram.length();i++){
state1.deltaWord(gram[i]);
}
state2.startWord(sorted_gram[0]);
for(unsigned int i=1;state2.isValid()&&i<sorted_gram.length();i++){
state2.deltaWord(sorted_gram[i]);
}
if(state1.isFinal() && state2.isFinal()){
unsigned int c1,c2;
c1=*((unsigned int*)state1.data());
c2=*((unsigned int*)state2.data());
std::cout << gram << "\t" << c1 << "," << c2 << "," << (double)c1/(double)c2 << std::endl;
}
}
}
return 0;
}
|
