fsa/queryproc/sort_grams.cpp


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include <iostream>
#include <iomanip>

#include "permuter.h"
#include "ngram.h"
#include "base64.h"
#include "wordchartokenizer.h"

using namespace fsa;

int main(int argc, char **argv)
{

  NGram query;
  WordCharTokenizer tokenizer(WordCharTokenizer::PUNCTUATION_WHITESPACEONLY);
  std::string qstr;

  while(!std::cin.eof()){
    getline(std::cin,qstr);
    query.set(qstr,tokenizer,0,-1);
    query.sort();
    query.uniq();
    std::cout << query << std::endl;
  }


  return 0;
}