1772 lines
52 KiB
C++
1772 lines
52 KiB
C++
/*
|
|
******************************************************************************
|
|
* Copyright (C) 2005-2006, International Business Machines Corporation and *
|
|
* others. All Rights Reserved. *
|
|
******************************************************************************
|
|
*/
|
|
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#include <stdlib.h>
|
|
#include <time.h>
|
|
|
|
#include "wbnf.h"
|
|
|
|
// Most of this code is meant to test the test code. It's a self test.
|
|
// Normally this isn't run.
|
|
#define TEST_WBNF_TEST 0
|
|
|
|
///////////////////////////////////////////////////////////
|
|
//
|
|
// Constants and the most basic helper classes
|
|
//
|
|
|
|
static const char DIGIT_CHAR[] = "0123456789";
|
|
static const char WHITE_SPACE[] = {'\t', ' ', '\r', '\n', 0};
|
|
static const char ALPHABET[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
|
static const char SPECIAL[] = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
|
|
|
|
static inline UBool isInList(const char c /*in*/, const char list[] /*in*/){
|
|
const char * p = list;
|
|
for (;*p != 0 && *p != c; p++);
|
|
return *p?TRUE:FALSE;
|
|
}
|
|
static inline UBool isDigit(char c) {return isInList(c, DIGIT_CHAR);}
|
|
static inline UBool isWhiteSpace(char c) {return isInList(c, WHITE_SPACE);}
|
|
static inline UBool isAlphabet(char c) {return isInList(c, ALPHABET);}
|
|
static inline UBool isSpecialAsciiChar(char c) {return isInList(c,SPECIAL);}
|
|
|
|
|
|
|
|
///////////////////////////////////////////////////////////
|
|
//
|
|
// Helper classes
|
|
//
|
|
|
|
class Buffer_byte{
|
|
// Utility class, can be treated as an auto expanded array. no boundary check.
|
|
|
|
typedef char byte;
|
|
byte * start;
|
|
byte * current;
|
|
int buffer_size; // size unit is byte
|
|
public:
|
|
inline int content_size(){return current - start;} // size unit is byte
|
|
|
|
private:
|
|
inline void expand(int add_size = 100){ // size unit is byte
|
|
int new_size = buffer_size + add_size;
|
|
|
|
int cs_snap = content_size();
|
|
start = (byte *) realloc(start, new_size); // may change the value of start
|
|
current = start + cs_snap;
|
|
|
|
memset(current, 0, add_size);
|
|
buffer_size = new_size;
|
|
}
|
|
|
|
inline void expand_to(int size){
|
|
int r = size - buffer_size;
|
|
if (r > 0) {
|
|
expand(r); // simply expand, no block alignment
|
|
}
|
|
}
|
|
Buffer_byte(const Buffer_byte &);
|
|
Buffer_byte & operator = (const Buffer_byte &);
|
|
public:
|
|
Buffer_byte():start(NULL),current(start),buffer_size(0){
|
|
expand();
|
|
}
|
|
~Buffer_byte(){
|
|
free(start);
|
|
}
|
|
|
|
inline void reset(){
|
|
start != NULL ? memset(start, 0, buffer_size) : 0;
|
|
current = start;
|
|
}
|
|
|
|
// Using memory copy method to append a C array to buffer,
|
|
inline void append(const void * c, int size){ // size unit is byte
|
|
expand_to(content_size() + size) ;
|
|
memcpy(current, c, size);
|
|
current = current + size;
|
|
}
|
|
|
|
byte * buffer(){
|
|
return start;
|
|
}
|
|
};
|
|
|
|
/*
|
|
The class(es) try to work as bulid-in array, so it overloads these two operators
|
|
operator type *();
|
|
type & operator[];
|
|
The first is used to auto type convert, the latter is used to select member.
|
|
|
|
A small trick is the class does not overload the address-of operator. This
|
|
behavior is different from bulid-in array, but it give us the opportunity
|
|
to get the address of the class itself.
|
|
*/
|
|
//template<typename type>
|
|
// class BUFFER{
|
|
// typedef BUFFER name;
|
|
#define BUFFER(type, name)\
|
|
class name {\
|
|
private:\
|
|
Buffer_byte buf;\
|
|
public:\
|
|
name & reset() {buf.reset(); return *this;}\
|
|
name & append(type c) {buf.append(&c, sizeof(type)); return *this;}\
|
|
name & append_array(const type * p, int size) {buf.append(p, sizeof(type)*size); return *this;}\
|
|
type & operator [] (int i) { return ((type *) buf.buffer())[i];}\
|
|
operator type *(){return (type *) buf.buffer();} \
|
|
int content_size(){return buf.content_size() / sizeof(type);}\
|
|
}
|
|
|
|
|
|
class Pick{
|
|
/* The Pick is the basic language generator element*/
|
|
public:
|
|
// generate a string accroding the syntax
|
|
// Return a null-terminated c-string. The buffer is owned by callee.
|
|
virtual const char* next() = 0;
|
|
virtual ~Pick(){};
|
|
};
|
|
|
|
//typedef BUFFER<char> Buffer_char;
|
|
//typedef BUFFER<int> Buffer_int;
|
|
//typedef BUFFER<Pick *> Buffer_pPick;
|
|
BUFFER(char, Buffer_char);
|
|
BUFFER(int, Buffer_int);
|
|
BUFFER(Pick *, Buffer_pPick);
|
|
|
|
class SymbolTable{
|
|
/* Helper class.
|
|
* It's a mapping table between 'variable name' and its 'active Pick object'
|
|
*/
|
|
private:
|
|
Buffer_char name_buffer; // var names storage space
|
|
|
|
Buffer_int names; // points to name (offset in name_buffer)
|
|
Buffer_pPick refs; // points to Pick
|
|
|
|
int get_index(const char *const var_name){
|
|
int len = names.content_size();
|
|
for (int i=0; i< len; i++){
|
|
if (strcmp(var_name, name_buffer + names[i]) == 0){
|
|
return i;
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
public:
|
|
enum RESULT {EMPTY, NO_VAR, NO_REF, HAS_REF};
|
|
|
|
RESULT find(const char *const var_name /*[in] c-string*/, Pick * * ref = NULL /*[out] Pick* */){
|
|
if (!var_name) return EMPTY; // NULL name
|
|
|
|
int i = get_index(var_name);
|
|
if (i == -1){
|
|
return NO_VAR; // new name
|
|
}
|
|
if (!refs[i]){ // exist name, no ref
|
|
return NO_REF;
|
|
} else {
|
|
if (ref) {
|
|
*ref = refs[i];
|
|
}
|
|
return HAS_REF; // exist name, has ref
|
|
}
|
|
}
|
|
|
|
void put(const char *const var_name, Pick *const var_ref = NULL){
|
|
int i = get_index(var_name);
|
|
switch(find(var_name)){
|
|
case EMPTY: // NULL name
|
|
break;
|
|
case NO_VAR: // new name
|
|
int offset;
|
|
offset = name_buffer.content_size();
|
|
name_buffer.append_array(var_name, strlen(var_name) + 1);
|
|
names.append(offset);
|
|
refs.append(var_ref);
|
|
break;
|
|
case NO_REF: // exist name, no ref
|
|
refs[i] = var_ref; // link definition with variable
|
|
break;
|
|
case HAS_REF: // exist name, has ref
|
|
if (var_ref){
|
|
refs[i] = var_ref;
|
|
}
|
|
break;
|
|
default:
|
|
; // ASSERT(FALSE);
|
|
}
|
|
return;
|
|
}
|
|
|
|
UBool is_complete(){
|
|
int n = names.content_size();
|
|
for (int i=0; i<n; ++i){
|
|
if (refs[i] == NULL){
|
|
return FALSE;
|
|
}
|
|
}
|
|
return TRUE;
|
|
}
|
|
|
|
void reset(){
|
|
names.reset();
|
|
name_buffer.reset();
|
|
|
|
// release memory here
|
|
int s = refs.content_size();
|
|
for (int i=0; i < s; i++){
|
|
delete refs[i]; // TOFIX: point alias/recursion problem
|
|
}
|
|
refs.reset();
|
|
}
|
|
|
|
~SymbolTable(){
|
|
reset();
|
|
}
|
|
};
|
|
|
|
|
|
/*
|
|
// Document of class Escaper
|
|
//
|
|
// ATTENTION:
|
|
// From http://icu.sourceforge.net/userguide/Collate_Customization.html.
|
|
// We get the precedence of escape/quote operations
|
|
//
|
|
// (highest) 1. backslash \
|
|
// 2. two single quotes ''
|
|
// 3. quoting ' '
|
|
//
|
|
// ICU Collation should accept following as the same string.
|
|
//
|
|
// 1) 'ab'c _
|
|
// 2) a\bc \
|
|
// 3) a'b'\c |- They are equal.
|
|
// 4) abc _/
|
|
//
|
|
// From "two single quotes", we have following deductions
|
|
// D1. empty quoting is illgal. (obviously)
|
|
// D2. no contact operation between two quotings
|
|
// '.''.' is not .. it is .'.
|
|
// D3. "two single quotes" cannot contact two quoting simultaneously
|
|
// '..''''.' is not ..'. it is ..''.
|
|
// NOTICE:
|
|
// "two single quotes" can contact before one quoting
|
|
// '''.' is '.
|
|
// "two single quotes" can literally contact after one quoting
|
|
// But, from syntax, it's one quoting including a "two single quotes"
|
|
// '.''' is .'
|
|
// D4. "two single quotes" cannot solely be included in quoting
|
|
// '''' is not ' it is ''
|
|
// NOTICE: These are legal
|
|
// '.''.' is .'.
|
|
// '.''' is .'
|
|
//
|
|
// dicision
|
|
// /\
|
|
// /__\
|
|
// output buffer input buffer
|
|
//
|
|
// To make our dicision (within an atom operation) without caring input and output buffer,
|
|
// following calling pattern (within an atom operation) shall be avoided
|
|
//
|
|
// P1 open_quoting() then close_quoting() (direct violation) D1
|
|
// P2 close_quoting() then open_quoting() (direct violation) D2
|
|
// P3 empty open_quoting() (indirect violation) D1, D4
|
|
// P4 empty close_quoting() (indirect violation) D2, D3
|
|
// P5 open_quoting() then two single quotes (indirect violation) D4
|
|
// P6 close_quoting() then two single quotes (indirect violation) D3
|
|
//
|
|
// two single quotes escaping will not open_ or close_ quoting()
|
|
// The choice will not lose some quoing forms.
|
|
//
|
|
// For open_quoting(),
|
|
// we may get this form quoting ''' P5
|
|
// It may raise a bug ''''x
|
|
// If we expect
|
|
// '''.' let the next char open the quoting
|
|
// '.''.' the quoting is already opened by preceding char
|
|
//
|
|
// For close_quoting()
|
|
// we will get this form quoting '.''' P6
|
|
// It may raise a bug '.''''.'
|
|
// If we expect
|
|
// '.'''\. let the next char close the quoting
|
|
// '.''''.' the expectation is wrong! using '.'\''.' instead
|
|
//
|
|
// It's a hard work to re-adjust generation opportunity for various escaping form.
|
|
// We just simply ignore it.
|
|
*/
|
|
class Escaper{
|
|
public:
|
|
enum CHOICE {YES, NO, RAND};
|
|
enum ESCAPE_FORM {BSLASH_ONLY, QUOTE_ONLY, QUOTE_AND_BSLAH, RAND_ESC};
|
|
private:
|
|
class Bool{ // A wrapper class for CHOICE, to auto adapter UBool class
|
|
private:
|
|
const CHOICE tag;
|
|
public:
|
|
Bool(CHOICE flag=RAND):tag(flag){}
|
|
operator UBool() { // conversion operator
|
|
return tag == RAND ? rand()%2 : tag == YES;
|
|
//if (tag == RAND){
|
|
// return rand()%2 == 1;
|
|
//} else {
|
|
// return tag == YES ? TRUE : FALSE;
|
|
//}
|
|
}
|
|
};
|
|
public:
|
|
Escaper(CHOICE escapeLiteral = RAND,
|
|
CHOICE twoQuotesEscape = RAND,
|
|
ESCAPE_FORM escapeForm = RAND_ESC):
|
|
escape_form(escapeForm),
|
|
escape_literal(escapeLiteral),
|
|
two_quotes_escape(twoQuotesEscape),
|
|
is_quoting(FALSE){}
|
|
private:
|
|
Buffer_char str;
|
|
ESCAPE_FORM escape_form;
|
|
Bool escape_literal;
|
|
Bool two_quotes_escape;
|
|
UBool quote_escape;
|
|
UBool bslash_escape;
|
|
UBool is_quoting;
|
|
|
|
void set_options(){
|
|
ESCAPE_FORM t = escape_form == RAND_ESC ? (ESCAPE_FORM) (rand()%3) : escape_form;
|
|
switch (t){
|
|
case BSLASH_ONLY :
|
|
bslash_escape = TRUE; quote_escape = FALSE; break;
|
|
case QUOTE_ONLY:
|
|
bslash_escape = FALSE;quote_escape = TRUE; break;
|
|
case QUOTE_AND_BSLAH:
|
|
bslash_escape = TRUE; quote_escape = TRUE; break;
|
|
default:
|
|
;// error
|
|
}
|
|
}
|
|
|
|
void reset(){
|
|
str.reset();
|
|
is_quoting = FALSE;
|
|
}
|
|
|
|
inline void open_quoting(){
|
|
if(is_quoting){
|
|
// do nothing
|
|
} else {
|
|
str.append('\'');
|
|
is_quoting = TRUE;
|
|
}
|
|
}
|
|
inline void close_quoting(){
|
|
if(is_quoting){
|
|
str.append('\'');
|
|
is_quoting = FALSE;
|
|
} else {
|
|
// do nothing
|
|
}
|
|
}
|
|
|
|
// str [in] null-terminated c-string
|
|
void append(const char * strToAppend){
|
|
for(;*strToAppend != 0; strToAppend++){
|
|
append(*strToAppend);
|
|
}
|
|
}
|
|
|
|
inline void append(const char c){
|
|
set_options();
|
|
|
|
if (c == '\\'){
|
|
quote_escape ? open_quoting() : close_quoting();
|
|
//bslash_escape always true here
|
|
str.append('\\');
|
|
str.append('\\');
|
|
} else if (c == '\''){
|
|
if (two_quotes_escape){ // quoted using two single quotes
|
|
// See documents in anonymous.design
|
|
str.append('\'');
|
|
str.append('\'');
|
|
} else{
|
|
quote_escape ? open_quoting() : close_quoting();
|
|
//bslash_escape always true here
|
|
str.append('\\');
|
|
str.append('\'');
|
|
}
|
|
} else if (isSpecialAsciiChar(c) || isWhiteSpace(c)){
|
|
quote_escape ? open_quoting() : close_quoting();
|
|
if (bslash_escape) str.append('\\');
|
|
str.append(c);
|
|
} else { //if (isAlphabet(c) || isDigit(c) || TRUE){ // treat others as literal
|
|
if (escape_literal){
|
|
quote_escape ? open_quoting() : close_quoting();
|
|
if (bslash_escape) str.append('\\');
|
|
str.append(c);
|
|
} else {
|
|
close_quoting();
|
|
str.append(c);
|
|
}
|
|
}
|
|
}
|
|
|
|
public:
|
|
// Return a null-terminate c-string. The buffer is owned by callee.
|
|
char * operator()(const char * literal /*c-string*/){
|
|
str.reset();
|
|
for(;*literal != 0; literal++){
|
|
append(*literal);
|
|
}
|
|
close_quoting(); // P4 exception, to close whole quoting
|
|
return str;
|
|
}
|
|
};
|
|
|
|
class WeightedRand{
|
|
// Return a random number in [0, size)
|
|
// Every number has different chance (aka weight) to be selected.
|
|
private:
|
|
Buffer_int weights;
|
|
double total;
|
|
WeightedRand(const WeightedRand &);
|
|
WeightedRand & operator = (const WeightedRand &);
|
|
public:
|
|
WeightedRand(Buffer_int * weight_list = NULL, int size = 0){
|
|
if ( weight_list == NULL){
|
|
for (int i=0; i<size; ++i) weights.append(DEFAULT_WEIGHT);
|
|
} else {
|
|
int s = weight_list->content_size();
|
|
if (s < size){
|
|
weights.append_array( (*weight_list),s);
|
|
for (int i=s; i<size; ++i) weights.append(DEFAULT_WEIGHT);
|
|
} else { // s >= size
|
|
weights.append_array( (*weight_list),size);
|
|
}
|
|
}
|
|
total = 0;
|
|
int c = weights.content_size();
|
|
for (int i=0; i<c; ++i){
|
|
total += weights[i];
|
|
}
|
|
}
|
|
|
|
void append(int weight){
|
|
weights.append(weight);
|
|
total += weight;
|
|
}
|
|
|
|
// Give a random number with the consideration of weight.
|
|
// Every random number is associated with a weight.
|
|
// It identifies the chance to be selected,
|
|
// larger weight has more chance to be selected.
|
|
//
|
|
//
|
|
// ______________________ every slot has equal chance
|
|
//
|
|
// [____][_][___][______] each item has different slots, hence different chance
|
|
//
|
|
//
|
|
// The algorithms to generate the number is illustrated by preceding figure.
|
|
// First, a slot is selected by rand(). Then we translate the slot to corresponding item.
|
|
//
|
|
int next(){
|
|
// get a random in [0,1]
|
|
double reference_mark = (double)rand() / (double)RAND_MAX;
|
|
|
|
// get the slot's index, 0 <= mark <= total;
|
|
double mark = total * reference_mark;
|
|
|
|
// translate the slot to corresponding item
|
|
int i=0;
|
|
for (;;){
|
|
mark -= weights[i]; // 0 <= mark <= total
|
|
if (mark <= 0)
|
|
break;
|
|
i++;
|
|
}
|
|
return i;
|
|
}
|
|
};
|
|
|
|
///////////////////////////////////////////////////////////
|
|
//
|
|
// The parser result nodes
|
|
//
|
|
|
|
class Literal : public Pick {
|
|
public:
|
|
virtual const char* next(){
|
|
return str;
|
|
}
|
|
Literal(const char * s /*c-string*/){
|
|
str.append_array(s, strlen(s) + 1);
|
|
}
|
|
private:
|
|
Buffer_char str; //null-terminated c-string
|
|
};
|
|
|
|
class Variable : public Pick {
|
|
public:
|
|
Variable(SymbolTable * symbols, const char * varName, Pick * varRef = NULL){
|
|
this->var_name.append_array(varName, strlen(varName) + 1);
|
|
if ((symbol_table = symbols)){
|
|
symbol_table->put(varName, varRef);
|
|
}
|
|
}
|
|
|
|
operator const char *(){
|
|
return var_name;
|
|
}
|
|
|
|
virtual const char* next(){
|
|
if (symbol_table){
|
|
Pick * var_ref = NULL;
|
|
symbol_table->find(var_name, &var_ref);
|
|
if (var_ref) {
|
|
return var_ref->next();
|
|
}
|
|
}
|
|
return ""; // dumb string
|
|
}
|
|
private:
|
|
Buffer_char var_name;
|
|
SymbolTable * symbol_table;
|
|
};
|
|
|
|
class Quote : public Pick{
|
|
public:
|
|
Quote(Pick & base):item(base),e(Escaper::NO, Escaper::NO, Escaper::BSLASH_ONLY){
|
|
}
|
|
virtual const char* next(){
|
|
return e(item.next());
|
|
}
|
|
private:
|
|
Pick & item;
|
|
Buffer_char str;
|
|
Escaper e;
|
|
};
|
|
|
|
|
|
class Morph : public Pick{
|
|
/*
|
|
The difference between morph and an arbitrary random string is that
|
|
a morph changes slowly. When we build collation rules, for example,
|
|
it is a much better test if the strings we use are all in the same
|
|
'neighborhood'; they share many common characters.
|
|
*/
|
|
public:
|
|
Morph(Pick & base):item(base){}
|
|
|
|
virtual const char* next(){
|
|
current.reset();
|
|
const char * s = item.next();
|
|
current.append_array(s, strlen(s) + 1);
|
|
if (last.content_size() == 0) {
|
|
str.reset();
|
|
last.reset();
|
|
str.append_array(current, current.content_size());
|
|
last.append_array(current, current.content_size());
|
|
} else {
|
|
morph();
|
|
}
|
|
return str;
|
|
}
|
|
private:
|
|
Pick & item;
|
|
Buffer_char str;
|
|
Buffer_char last;
|
|
Buffer_char current;
|
|
|
|
char * p_last;
|
|
char * p_curr;
|
|
|
|
void copy_curr(){
|
|
if (*p_curr) {
|
|
str.append(*p_curr);
|
|
p_curr++;
|
|
}
|
|
}
|
|
|
|
void copy_last(){
|
|
if (*p_last) {
|
|
str.append(*p_last);
|
|
p_last++;
|
|
}
|
|
}
|
|
|
|
// copy 0, 1, or 2 character(s) to str
|
|
void copy(){
|
|
static WeightedRand wr(& Buffer_int().append(DEFAULT_WEIGHT * 10), 5);
|
|
|
|
switch (wr.next()){
|
|
case 0: // copy last -- has 10 times chance than others
|
|
copy_last();
|
|
break;
|
|
case 1: // copy both
|
|
copy_curr();
|
|
copy_last();
|
|
break;
|
|
case 2: // copy both
|
|
copy_last();
|
|
copy_curr();
|
|
break;
|
|
case 3:
|
|
copy_curr();
|
|
break;
|
|
case 4: // copy nothing
|
|
break;
|
|
default:
|
|
// ASSERT(FALSE);
|
|
;
|
|
}
|
|
}
|
|
|
|
void morph(void){
|
|
int min = strlen(last);
|
|
int max = strlen(current);
|
|
if (min > max){
|
|
int temp = min;
|
|
min = max;
|
|
max = temp;
|
|
}
|
|
|
|
int len = min + rand()%(max - min + 1); // min + [0, diff]
|
|
p_curr = current;
|
|
p_last = last;
|
|
str.reset();
|
|
|
|
for (; str.content_size()<len && *p_curr && *p_last;){
|
|
copy(); // copy 0, 1, or 2 character(s) to str
|
|
}
|
|
|
|
if (str.content_size() == len) {
|
|
str.append(0);
|
|
final();
|
|
return;
|
|
}
|
|
|
|
if (str.content_size() > len) { // if the last copy copied two characters
|
|
str[len]=0;
|
|
final();
|
|
return;
|
|
}
|
|
|
|
// str.content_size() < len
|
|
if (*p_last) {
|
|
for (; str.content_size() < len; copy_last());
|
|
} else if (*p_curr){
|
|
for (; str.content_size() < len; copy_curr());
|
|
}
|
|
|
|
int last_len = last.content_size();
|
|
for (;str.content_size() < len;){
|
|
str.append(last[rand()%last_len]);
|
|
}
|
|
str.append(0);
|
|
final();
|
|
}
|
|
|
|
void final(){
|
|
last.reset();
|
|
last.append_array(current, current.content_size());
|
|
}
|
|
};
|
|
|
|
class Sequence : public Pick {
|
|
public:
|
|
virtual const char* next(){
|
|
str.reset();
|
|
int s = items.content_size();
|
|
for(int i=0; i < s; i++){
|
|
const char * t = items[i]->next();
|
|
str.append_array(t, strlen(t));
|
|
}
|
|
str.append(0); // terminal null
|
|
return str;
|
|
}
|
|
|
|
void append (Pick * node){
|
|
items.append(node);
|
|
}
|
|
|
|
virtual ~Sequence(){
|
|
int s = items.content_size();
|
|
for(int i=0; i < s; i++){
|
|
//How can assure the item is got from heap?
|
|
//Let's assume it.
|
|
delete items[i]; // TOFIX: point alias/recursion problem
|
|
items[i] = NULL;
|
|
}
|
|
}
|
|
private:
|
|
Buffer_pPick items;
|
|
Buffer_char str; //null-terminated c-string
|
|
};
|
|
|
|
class Repeat : public Pick {
|
|
private:
|
|
Pick * item;
|
|
Buffer_char str;
|
|
WeightedRand wr;
|
|
int min;
|
|
int max;
|
|
int select_a_count(){
|
|
return min + wr.next();
|
|
}
|
|
public:
|
|
virtual const char* next(){
|
|
str.reset();
|
|
int c = select_a_count();
|
|
for(int i=0; i< c; i++){
|
|
const char * t = item->next();
|
|
str.append_array(t, strlen(t));
|
|
}
|
|
str.append(0);
|
|
return str;
|
|
}
|
|
|
|
Repeat(Pick * base, int minCount =0, int maxCount = 1, Buffer_int * weights = NULL):
|
|
wr(weights, maxCount-minCount +1) {
|
|
this->item = base;
|
|
this->min = minCount;
|
|
this->max = maxCount;
|
|
}
|
|
virtual ~Repeat(){
|
|
delete item; // TOFIX: point alias/recursion problem
|
|
item = NULL;
|
|
}
|
|
};
|
|
|
|
|
|
class Alternation : public Pick {
|
|
public:
|
|
virtual const char* next(){
|
|
str.reset();
|
|
int i = wr.next();
|
|
const char * t = items[i]->next();
|
|
str.append_array(t, strlen(t) + 1);
|
|
return str;
|
|
}
|
|
virtual ~Alternation(){
|
|
int s = items.content_size();
|
|
for(int i=0; i < s; i++){
|
|
delete items[i]; // TOFIX: point alias/recursion problem
|
|
items[i] = NULL;
|
|
}
|
|
}
|
|
|
|
Alternation & append (Pick * node, int weight = DEFAULT_WEIGHT){
|
|
items.append(node);
|
|
wr.append(weight);
|
|
return *this;
|
|
}
|
|
private:
|
|
Buffer_pPick items;
|
|
Buffer_char str; // null-terminated c-string
|
|
WeightedRand wr;
|
|
};
|
|
|
|
///////////////////////////////////////////////////////////
|
|
//
|
|
// The parser
|
|
//
|
|
|
|
enum TokenType {STRING, VAR, NUMBER, STREAM_END, ERROR, QUESTION, STAR, PLUS, LBRACE, RBRACE, LPAR, RPAR, SEMI, EQ, COMMA, BAR, AT, WAVE, PERCENT};
|
|
|
|
class Scanner{
|
|
friend int DumpScanner(Scanner & s, UBool dumb);
|
|
private:
|
|
const char * source;
|
|
const char * working;
|
|
const char * history; // for debug
|
|
enum StateType {START, IN_NUM, IN_VAR_FIRST, IN_VAR, IN_QUOTE, IN_QUOTE_BSLASH, IN_BSLASH, IN_STRING, DONE};
|
|
StateType state;
|
|
void terminated(TokenType t){
|
|
working--; // return the peeked character
|
|
tokenType = t;
|
|
token.append(0); // close buffer
|
|
state = DONE;
|
|
}
|
|
public:
|
|
// the buffer of "source" is owned by caller
|
|
Scanner(const char *src/*[in] c-string*/ = NULL):source(src){
|
|
working = src;
|
|
history = working;
|
|
state = DONE;
|
|
tokenType = ERROR;
|
|
}
|
|
|
|
//void setSource(const char *const src /*[in] c-string*/){
|
|
// *(&const_cast<const char *>(source)) = src;
|
|
//}
|
|
|
|
Buffer_char token;
|
|
TokenType tokenType;
|
|
|
|
TokenType getNextToken(){
|
|
token.reset();
|
|
state = START;
|
|
history = working; // for debug
|
|
while (state != DONE){
|
|
char c = *working++;
|
|
if (c == 0 && state != START){//avoid buffer overflow. for IN_QUOE, IN_ESCAPE
|
|
terminated(ERROR);
|
|
break; // while
|
|
}
|
|
switch(state){
|
|
case START:
|
|
tokenType = ERROR;
|
|
switch(c){
|
|
case '?' : tokenType = QUESTION; break;
|
|
case '*' : tokenType = STAR; break;
|
|
case '+' : tokenType = PLUS; break;
|
|
case '{' : tokenType = LBRACE; break;
|
|
case '}' : tokenType = RBRACE; break;
|
|
case '(' : tokenType = LPAR; break;
|
|
case ')' : tokenType = RPAR; break;
|
|
case ';' : tokenType = SEMI; break;
|
|
case '=' : tokenType = EQ; break;
|
|
case ',' : tokenType = COMMA; break;
|
|
case '|' : tokenType = BAR; break;
|
|
case '@' : tokenType = AT; break;
|
|
case '~' : tokenType = WAVE; break;
|
|
case '%' : tokenType = PERCENT; break;
|
|
case 0 : tokenType = STREAM_END; working-- /*avoid buffer overflow*/; break;
|
|
}
|
|
if (tokenType != ERROR){
|
|
token.append(c);
|
|
token.append(0);
|
|
state = DONE;
|
|
break; // START
|
|
}
|
|
switch(c){
|
|
case '$' : state = IN_VAR_FIRST; token.append(c); break;
|
|
case '\'' : state = IN_QUOTE; break;
|
|
case '\\' : state = IN_BSLASH; break;
|
|
default:
|
|
if (isWhiteSpace(c)){ // state = START; //do nothing
|
|
} else if (isDigit(c)){ state = IN_NUM; token.append(c);
|
|
} else if (isAlphabet(c)){ state = IN_STRING; token.append(c);
|
|
} else {terminated(ERROR);}
|
|
}
|
|
break;//START
|
|
case IN_NUM:
|
|
if (isDigit(c)){
|
|
token.append(c);
|
|
} else {
|
|
terminated(NUMBER);
|
|
}
|
|
break;//IN_NUM
|
|
case IN_VAR_FIRST:
|
|
if (isAlphabet(c)){
|
|
token.append(c);
|
|
state = IN_VAR;
|
|
} else {
|
|
terminated(ERROR);
|
|
}
|
|
break; // IN_VAR_FISRT
|
|
case IN_VAR:
|
|
if (isAlphabet(c) || isDigit(c)){
|
|
token.append(c);
|
|
} else {
|
|
terminated(VAR);
|
|
}
|
|
break;//IN_VAR
|
|
case IN_STRING:
|
|
// About the scanner's behavior for STRING, AT, and ESCAPE:
|
|
// All of them can be contacted with each other.
|
|
// This means the scanner will eat up as much as possible strings
|
|
// (STRING, AT, and ESCAPE) at one time, with no regard of their
|
|
// combining sequence.
|
|
//
|
|
if (c == '\''){
|
|
state = IN_QUOTE; // the first time we see single quote
|
|
} else if (c =='\\'){ // back slash character
|
|
state = IN_BSLASH;
|
|
} else if (isAlphabet(c) || isDigit(c)){
|
|
token.append(c);
|
|
} else{
|
|
terminated(STRING);
|
|
}
|
|
break;//IN_STRING
|
|
case IN_QUOTE:
|
|
if (c == '\''){ // the second time we see single quote
|
|
state = IN_STRING; // see document in IN_STRING
|
|
} else if ( c== '\\') { // backslah escape in quote
|
|
state = IN_QUOTE_BSLASH;
|
|
} else {
|
|
token.append(c); // eat up everything, includes back slash
|
|
}
|
|
break;//IN_QUOTE
|
|
case IN_QUOTE_BSLASH:
|
|
case IN_BSLASH:
|
|
switch (c){
|
|
case 'n' : token.append('\n'); break;
|
|
case 'r' : token.append('\r'); break;
|
|
case 't' : token.append('\t'); break;
|
|
case '\'' : token.append('\''); break;
|
|
case '\\' : token.append('\\'); break;
|
|
default: token.append(c); // unknown escaping, treat it as literal
|
|
}
|
|
if (state == IN_BSLASH){
|
|
state = IN_STRING; // see document in IN_STRING
|
|
} else { // state == IN_QUOTE_BSLASH
|
|
state = IN_QUOTE;
|
|
}
|
|
break;//IN_BSLASH
|
|
case DONE: /* should never happen */
|
|
default:
|
|
working--;
|
|
tokenType = ERROR;
|
|
state = DONE;
|
|
break;
|
|
}//switch(state)
|
|
}//while (state != DONE)
|
|
|
|
return tokenType;
|
|
}
|
|
};//class Scanner
|
|
|
|
class Parser{
|
|
friend UBool TestParser();
|
|
friend class TestParserT;
|
|
friend class LanguageGenerator_impl;
|
|
private:
|
|
Scanner s;
|
|
TokenType & token;
|
|
int min_max; // for the evil infinite
|
|
|
|
UBool match(TokenType expected){
|
|
if (token == expected) {
|
|
token = s.getNextToken();
|
|
return TRUE;
|
|
} else {
|
|
//s.dumpCurrentPoint();
|
|
return FALSE;
|
|
}
|
|
}
|
|
|
|
UBool weight(int & value){
|
|
if (token == NUMBER){
|
|
int temp = atoi(s.token);
|
|
match(NUMBER);
|
|
if (match(PERCENT)){
|
|
value = temp;
|
|
return TRUE;
|
|
}
|
|
}
|
|
return FALSE;
|
|
}
|
|
|
|
UBool repeat (Pick* &node /*in,out*/){
|
|
if (node == NULL) return FALSE;
|
|
|
|
int count = -2;
|
|
int min = -2;
|
|
int max = -2;
|
|
UBool question = FALSE;
|
|
switch (token){
|
|
case QUESTION:
|
|
match(QUESTION);
|
|
min = 0;
|
|
max = 1;
|
|
count = 2;
|
|
question = TRUE;
|
|
break;
|
|
case STAR:
|
|
match(STAR);
|
|
min = 0;
|
|
max = -1;
|
|
count = -1;
|
|
break;
|
|
case PLUS:
|
|
match(PLUS);
|
|
min = 1;
|
|
max = -1;
|
|
count = -1;
|
|
break;
|
|
case LBRACE:
|
|
match(LBRACE);
|
|
if (token != NUMBER){
|
|
return FALSE;
|
|
}else {
|
|
min = atoi(s.token);
|
|
match(NUMBER);
|
|
if (token == RBRACE){
|
|
match(RBRACE);
|
|
max = min;
|
|
count = 1;
|
|
} else if (token == COMMA) {
|
|
match(COMMA);
|
|
if (token == RBRACE){
|
|
match(RBRACE);
|
|
max = -1;
|
|
count = -1;
|
|
} else if (token == NUMBER) {
|
|
max = atoi(s.token);
|
|
match(NUMBER);
|
|
count = max - min + 1;
|
|
if (!match(RBRACE)) {
|
|
return FALSE;
|
|
}
|
|
} else {
|
|
return FALSE;
|
|
}
|
|
} else {
|
|
return FALSE;
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
return FALSE;
|
|
}
|
|
|
|
if (count == -2 || min == -2 || max == -2){
|
|
//ASSERT(FALSE);
|
|
return FALSE;
|
|
}
|
|
|
|
// eat up following weights
|
|
Buffer_int weights;
|
|
int w;
|
|
while (weight(w)){
|
|
weights.append(w);
|
|
}
|
|
|
|
// for the evil infinite
|
|
min_max = min_max > min ? min_max : min;
|
|
min_max = min_max > max ? min_max : max;
|
|
if (min_max > PSEUDO_INFINIT){
|
|
return FALSE; // PSEUDO_INFINIT is less than the real maximum
|
|
}
|
|
if (max == -1){ // the evil infinite
|
|
max = PSEUDO_INFINIT;
|
|
}
|
|
// for the strange question mark
|
|
if (question && weights.content_size() > 0){
|
|
Buffer_int w2;
|
|
w2.append(DEFAULT_WEIGHT - weights[0]).append(weights[0]);
|
|
node = new Repeat(node,min,max,&w2);
|
|
return TRUE;
|
|
}
|
|
node = new Repeat(node,min,max,&weights);
|
|
return TRUE;
|
|
}
|
|
|
|
UBool core(Pick* &node /*out*/){
|
|
if (node != NULL) return FALSE; //assert node == NULL
|
|
|
|
switch(token){
|
|
case LPAR:
|
|
match(LPAR);
|
|
if(defination(node) && match(RPAR)){
|
|
return TRUE;
|
|
}
|
|
return FALSE;
|
|
case VAR:
|
|
node = new Variable(&symbols, s.token);
|
|
match(VAR);
|
|
return TRUE;
|
|
case STRING:
|
|
node = new Literal(s.token);
|
|
match(STRING);
|
|
return TRUE;
|
|
default:
|
|
return FALSE;
|
|
}
|
|
}
|
|
UBool modified(Pick* &node /*out*/){
|
|
if (node != NULL) return FALSE; //assert node == NULL
|
|
|
|
if (!core(node)) {
|
|
return FALSE;
|
|
}
|
|
|
|
for (;;){
|
|
switch(token){
|
|
case WAVE:
|
|
match(WAVE);
|
|
node = new Morph(*node);
|
|
break;
|
|
case AT:
|
|
match(AT);
|
|
node = new Quote(*node);
|
|
break;
|
|
case QUESTION:
|
|
case STAR:
|
|
case PLUS:
|
|
case LBRACE:
|
|
if (!repeat(node)) return FALSE;
|
|
break;
|
|
case SEMI: // rule definiation closed
|
|
case RPAR: // within parenthesis (core closed)
|
|
case BAR: // in alternation
|
|
case NUMBER: // in alternation, with weight
|
|
case LPAR: // in sequence
|
|
case VAR: // in sequence
|
|
case STRING: // in sequence
|
|
return TRUE;
|
|
default:
|
|
return FALSE;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
UBool sequence_list(Pick* &node /*in,out*/){
|
|
if (node == NULL) return FALSE; // assert node != NULL
|
|
|
|
Sequence* seq = new Sequence();
|
|
Pick * n = node;
|
|
|
|
while (token == VAR || token == STRING || token == LPAR){
|
|
seq->append(n);
|
|
n = NULL;
|
|
if (modified(n)){
|
|
// go on
|
|
} else {
|
|
goto FAIL;
|
|
}
|
|
}
|
|
|
|
if (token == SEMI || token == RPAR || token == BAR){
|
|
seq->append(n);
|
|
node = seq;
|
|
return TRUE;
|
|
}
|
|
FAIL:
|
|
delete seq;
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
UBool sequence(Pick* &node /*out*/){
|
|
if (node != NULL) return FALSE; //assert node == NULL
|
|
|
|
if (!modified(node)) {
|
|
return FALSE;
|
|
}
|
|
|
|
if (token == VAR || token == STRING || token == LPAR){
|
|
return sequence_list(node);
|
|
} else {
|
|
return TRUE; // just a modified
|
|
}
|
|
}
|
|
|
|
UBool alternation_list(Pick* &node /*in,out*/){
|
|
if (node == NULL) return FALSE; // assert node != NULL
|
|
|
|
Alternation * alt = new Alternation();
|
|
Pick * n = node;
|
|
int w = DEFAULT_WEIGHT;
|
|
|
|
while (token == NUMBER || token == BAR){
|
|
if(token == NUMBER) {
|
|
if (weight(w)){
|
|
if (token == BAR){
|
|
// the middle item, go on
|
|
} else {
|
|
// the last item or encounter error
|
|
break; //while
|
|
}
|
|
} else {
|
|
goto FAIL;
|
|
}
|
|
} // else token == BAR
|
|
match(BAR);
|
|
alt->append(n,w);
|
|
|
|
n = NULL;
|
|
w = DEFAULT_WEIGHT;
|
|
if (sequence(n)){
|
|
// go on
|
|
} else {
|
|
goto FAIL;
|
|
}
|
|
}
|
|
|
|
if (token == SEMI || token == RPAR) {
|
|
alt->append(n,w);
|
|
node = alt;
|
|
return TRUE;
|
|
}
|
|
FAIL:
|
|
delete alt;
|
|
return FALSE;
|
|
}
|
|
|
|
UBool alternation(Pick* &node /*out*/){
|
|
if (node != NULL) return FALSE; //assert node == NULL
|
|
|
|
// 'sequence' has higher precedence than 'alternation'
|
|
if (!sequence(node)){
|
|
return FALSE;
|
|
}
|
|
|
|
if (token == BAR || token == NUMBER){ // find a real alternation1, create it.
|
|
return alternation_list(node);
|
|
} else {
|
|
return TRUE; // just a sequence_old
|
|
}
|
|
}
|
|
|
|
|
|
UBool defination(Pick* &node /*out*/){
|
|
if (node != NULL) return FALSE; //assert node == NULL
|
|
return alternation(node);
|
|
}
|
|
|
|
UBool rule(){
|
|
if (token == VAR){
|
|
Buffer_char name;
|
|
name.append_array(s.token, strlen(s.token) + 1);
|
|
match(VAR);
|
|
|
|
if (match(EQ)){
|
|
Pick * t = NULL;
|
|
if(defination(t)){
|
|
symbols.put(name, t);
|
|
return match(SEMI);
|
|
}
|
|
}
|
|
}
|
|
return FALSE;
|
|
}
|
|
public:
|
|
UBool rules(){
|
|
symbols.reset();
|
|
token = s.getNextToken();
|
|
while (rule()){
|
|
}
|
|
if (token == STREAM_END){
|
|
return TRUE;
|
|
} else {
|
|
//s.dumpCurrentPoint();
|
|
return FALSE;
|
|
}
|
|
}
|
|
|
|
public:
|
|
SymbolTable symbols;
|
|
|
|
Parser(const char *const source):s(source), token(s.tokenType){
|
|
min_max = -2;
|
|
}
|
|
UBool parse(){
|
|
return rules();
|
|
}
|
|
|
|
}; // class Parser
|
|
|
|
|
|
///////////////////////////////////////////////////////////
|
|
//
|
|
//
|
|
//
|
|
|
|
int DumpScanner(Scanner & s, UBool dump = TRUE){
|
|
int len = strlen(s.source);
|
|
int error_start_offset = s.history - s.source;
|
|
if (dump){
|
|
printf("\n=================== DumpScanner ================\n");
|
|
fwrite(s.source, len, 1, stdout);
|
|
printf("\n-----parsed-------------------------------------\n");
|
|
fwrite(s.source, s.history - s.source, 1, stdout);
|
|
printf("\n-----current------------------------------------\n");
|
|
fwrite(s.history, s.working - s.history, 1, stdout);
|
|
printf("\n-----unparsed-----------------------------------\n");
|
|
fwrite(s.working, (s.source + len - s.working), 1, stdout);
|
|
printf("\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n");
|
|
}
|
|
return error_start_offset;
|
|
}
|
|
|
|
class LanguageGenerator_impl{
|
|
public:
|
|
LanguageGenerator_impl(const char *const bnf_definition, const char *const top_node)
|
|
:par(bnf_definition), top_node_name(top_node){
|
|
srand((unsigned)time( NULL ));
|
|
}
|
|
|
|
LanguageGenerator::PARSE_RESULT parseBNF(UBool debug = TRUE){
|
|
if (par.parse()){
|
|
if (par.symbols.find(top_node_name, &top_node_ref) == SymbolTable::HAS_REF) {
|
|
if (par.symbols.is_complete()) {
|
|
return LanguageGenerator::OK;
|
|
} else {
|
|
if (debug) printf("The bnf definition is incomplete.\n");
|
|
return LanguageGenerator::INCOMPLETE;
|
|
}
|
|
} else {
|
|
if (debug) printf("No top node is found.\n");
|
|
return LanguageGenerator::NO_TOP_NODE;
|
|
}
|
|
} else {
|
|
if(debug) {
|
|
printf("The bnf definition is wrong\n");
|
|
DumpScanner(par.s, TRUE);
|
|
}
|
|
return LanguageGenerator::BNF_DEF_WRONG;
|
|
}
|
|
}
|
|
const char * next(){
|
|
return top_node_ref->next();
|
|
}
|
|
|
|
private:
|
|
Parser par;
|
|
const char *const top_node_name;
|
|
Pick * top_node_ref;
|
|
};
|
|
|
|
LanguageGenerator::LanguageGenerator():lang_gen(NULL){
|
|
}
|
|
|
|
LanguageGenerator::~LanguageGenerator(){
|
|
delete lang_gen;
|
|
}
|
|
|
|
LanguageGenerator::PARSE_RESULT LanguageGenerator::parseBNF(const char *const bnf_definition /*in*/, const char *const top_node/*in*/, UBool debug){
|
|
if (lang_gen){
|
|
delete lang_gen;
|
|
}
|
|
lang_gen = new LanguageGenerator_impl(bnf_definition, top_node);
|
|
PARSE_RESULT r = lang_gen->parseBNF(debug);
|
|
if (r != OK){
|
|
delete lang_gen;
|
|
lang_gen = NULL;
|
|
return r;
|
|
} else {
|
|
return r;
|
|
}
|
|
}
|
|
const char *LanguageGenerator::next(){ // Return a null-terminated c-string. The buffer is owned by callee.
|
|
if (lang_gen){
|
|
return lang_gen->next();
|
|
}else {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
///////////////////////////////////////////////////////////
|
|
//
|
|
// The test code for WBNF
|
|
//
|
|
|
|
#define CALL(fun) \
|
|
if (fun()){ \
|
|
printf("Pass: " #fun "\n");\
|
|
} else { \
|
|
printf("FAILED: !!! " #fun " !!!\n"); \
|
|
}
|
|
|
|
#define DUMP_R(fun, var, times) \
|
|
{printf("\n========= " #fun " =============\n"); \
|
|
for (int i=0; i<times; i++) { \
|
|
const char * t = var.next();\
|
|
fwrite(t,strlen(t),1,stdout); \
|
|
printf("\n"); \
|
|
} \
|
|
printf("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n");}
|
|
|
|
|
|
|
|
#if TEST_WBNF_TEST
|
|
static UBool TestQuote(){
|
|
const char *const str = "This ' A !,z| qq [] .new\tline";
|
|
//const char *const str_r = "This \\' A '!,'z'|' qq '[]' '.'new\tline";
|
|
////
|
|
//// :( we must quote our string to following C syntax
|
|
//// cannot type the literal here, it makes our code rather human unreadable
|
|
//// very very unconformable!
|
|
////
|
|
///*
|
|
//*/
|
|
|
|
//const char *const s1 = "ab'c";
|
|
//const char (* s1_r1) [] = { "ab''c", // ab''c
|
|
// "ab\\'c", // ab\'c
|
|
// };//
|
|
///*
|
|
// . '.' \.
|
|
// .. \.\. '.'\. '.'\. '..' // '.''.' wrong
|
|
//*/
|
|
|
|
//const char *const s2 = "a..'.b"; // a..'.b
|
|
//const char (*s2_r) [] = { "a'..''.'b" // a'..''.'b
|
|
// ,"a'..\\'.'b" // a'..\'.'b
|
|
// ,"a'..'\\''.'b" // a'..'\''.'b
|
|
// };//
|
|
|
|
//const char *const s3 = "a..\\.b"; // a..\.b
|
|
//const char (*s3_r) [] = { "a'..\\\\.'b" // a'..\\.'b
|
|
// ,"a'..'\\\\'.'b" // a'..'\\'.'b
|
|
// };//
|
|
|
|
// // no catact operation, no choice, must be compact
|
|
|
|
srand((unsigned)time( NULL ));
|
|
|
|
//Escaper l(Escaper::NO, Escaper::NO, Escaper::RAND_ESC);
|
|
Pick *p = new Literal(str);
|
|
Quote q(*p);
|
|
|
|
DUMP_R(TestQuote, (*p), 1);
|
|
DUMP_R(TestQuote, q, 20);
|
|
return FALSE;
|
|
}
|
|
static UBool TestLiteral(){
|
|
const char * s = "test string99.";
|
|
Literal n(s);
|
|
const char * r = n.next();
|
|
return strcmp(s,r) == 0;
|
|
}
|
|
|
|
static UBool TestSequence(){
|
|
Sequence seq;
|
|
seq.append(new Literal("abc "));
|
|
seq.append(new Literal(", s"));
|
|
|
|
return strcmp(seq.next(), "abc , s") == 0;
|
|
}
|
|
static UBool TestAlternation(){
|
|
srand((unsigned)time( NULL ));
|
|
Alternation alt;
|
|
alt.append(new Literal("aaa_10%"),10);
|
|
alt.append(new Literal("bbb_0%"),0);
|
|
alt.append(new Literal("ccc_10%"),10);
|
|
alt.append(new Literal("ddddddd_50%"),50);
|
|
|
|
DUMP_R(TestAlternation, alt, 50);
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
static UBool TestBuffer(){
|
|
Buffer_int t;
|
|
t.append(1).append(0).append(5);
|
|
int s = t.content_size();
|
|
for (int i=0; i<s; ++i){
|
|
printf("%d\n", t[i]);
|
|
}
|
|
return FALSE;
|
|
}
|
|
|
|
static UBool TestWeightedRand(){
|
|
srand((unsigned)time( NULL ));
|
|
Buffer_int t;
|
|
t.append(1).append(0).append(5);
|
|
WeightedRand wr(&Buffer_int().append(10).append(0).append(50),4);
|
|
// WeightedRand wr(&t,3);
|
|
for (int i=0; i< 50; ++i){
|
|
printf("%d\n", wr.next());
|
|
}
|
|
return FALSE;
|
|
}
|
|
|
|
static UBool TestRepeat(){
|
|
srand((unsigned)time( NULL ));
|
|
Repeat rep(new Literal("aaa1-5 "), 1, 5);
|
|
DUMP_R(TestRepeat, rep, 50);
|
|
|
|
Repeat r2(new Literal("b{1,3}1%0%5% "), 1, 3, &Buffer_int().append(1).append(0).append(5));
|
|
DUMP_R(TestRepeat, r2, 50);
|
|
|
|
Repeat r3(new Literal("aaa5-5 "), 5, 5);
|
|
DUMP_R(TestRepeat, r3, 50);
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
static UBool TestVariable(){
|
|
SymbolTable tab;
|
|
Pick * value = new Literal("string1");
|
|
Variable var1(&tab, "x", value);
|
|
|
|
Variable var2(&tab, "y");
|
|
// tab.put(var2, value); // TOFIX: point alias/recursion problem
|
|
Pick * value2 = new Literal("string2");
|
|
tab.put(var2, value2);
|
|
|
|
Pick * value3 = new Literal("string3");
|
|
Variable var3(&tab, "z");
|
|
tab.put("z", value3);
|
|
|
|
UBool pass;
|
|
pass = strcmp(var1.next(), value->next()) == 0;
|
|
pass = pass && strcmp(var2.next(), value2->next()) == 0;
|
|
pass = pass && strcmp(var3.next(), value3->next()) == 0;
|
|
return pass;
|
|
}
|
|
|
|
static UBool TestSymbolTable(){
|
|
Literal * n1 = new Literal("string1");
|
|
Literal * n2 = new Literal("string2");
|
|
SymbolTable t;
|
|
t.put("abc", n1);
|
|
t.put("$aaa", n2);
|
|
// t.put("alias", n1); // TOFIX: point alias/recursion problem
|
|
t.put("bbb");
|
|
|
|
UBool pass;
|
|
pass = t.find(NULL) == SymbolTable::EMPTY;
|
|
pass = pass && t.find("ccc") == SymbolTable::NO_VAR;
|
|
pass = pass && t.find("bbb") == SymbolTable::NO_REF;
|
|
pass = pass && t.find("abc") == SymbolTable::HAS_REF;
|
|
pass = pass && t.find("$aaa") == SymbolTable::HAS_REF;
|
|
|
|
t.reset();
|
|
pass = pass && t.find("abc") == SymbolTable::NO_VAR;
|
|
return pass;
|
|
}
|
|
|
|
|
|
static UBool TestScanner(void){
|
|
//const char str1[] = "$root = $command{0,5} $reset $mostRules{1,20};";
|
|
//const char str1_r[][20] = {"$root", "=", "$command", "{", "0", ",", "5", "}",
|
|
// "$reset", "$mostRules", "{", "1", ",", "20", "}", ";"};
|
|
|
|
const char str2[] = "$p2 =(\\\\ $s $string $s)? 25%;";
|
|
const char str2_r[][20] = {"$p2", "=", "(", "\\", "$s", "$string", "$s", ")", "?", "25", "%", ";"};
|
|
|
|
const char *str = str2;
|
|
const char (*str_r)[20] = str2_r;
|
|
int tokenNum = sizeof(str2_r)/sizeof(char[20]);
|
|
|
|
Scanner t(str);
|
|
UBool pass = TRUE;
|
|
t.getNextToken();
|
|
int i = 0;
|
|
while (pass){
|
|
if (t.tokenType == STREAM_END){
|
|
pass = pass? i == tokenNum : FALSE;
|
|
break;//while
|
|
} else if (t.tokenType == ERROR){
|
|
pass = FALSE;
|
|
break;//while
|
|
} else {
|
|
pass = strcmp( &(t.token[0]), str_r[i++]) == 0;
|
|
t.getNextToken();
|
|
}
|
|
}
|
|
|
|
//const char ts[] = "$commandList = '['"
|
|
//" ( alternate ' ' $alternateOptions"
|
|
//" | backwards ' 2'"
|
|
//" | normalization ' ' $onoff "
|
|
//" | caseLevel ' ' $onoff "
|
|
//" | hiraganaQ ' ' $onoff"
|
|
//" | caseFirst ' ' $caseFirstOptions"
|
|
//" | strength ' ' $strengthOptions"
|
|
//" ) ']';" ;
|
|
|
|
//Scanner t2(ts);
|
|
//pass = TRUE;
|
|
//do {
|
|
// t2.getNextToken();
|
|
// if (t2.tokenType == ERROR){
|
|
// DumpScanner(t2);
|
|
// return FALSE;
|
|
// }
|
|
//}while (t.tokenType != STREAM_END);
|
|
|
|
return pass;
|
|
}
|
|
|
|
class TestParserT {
|
|
public:
|
|
UBool operator () (const char *const str, const int exp_error_offset = -1, const UBool dump = TRUE){
|
|
Parser par(str);
|
|
if (par.rules()){
|
|
if ( exp_error_offset == -1){
|
|
return TRUE;
|
|
}else {
|
|
DumpScanner(par.s,dump);
|
|
return FALSE;
|
|
}
|
|
}else {
|
|
return DumpScanner(par.s, dump) == exp_error_offset;
|
|
}
|
|
}
|
|
};
|
|
|
|
UBool TestParser(){
|
|
TestParserT test;
|
|
|
|
UBool pass = TRUE;
|
|
pass = pass && test ("$s = ' ' ? 50%;");
|
|
pass = pass && test("$x = ($var {1,2}) 3%;"); // legal
|
|
pass = pass && test("$x = $var {1,2} 3% | b 4%;"); // legal
|
|
pass = pass && test("$x = $var {1,2} 3%;"); // legal
|
|
pass = pass && test("$m = $c ? 2% 4% | $r 5% | $n 25%;"); // legal
|
|
pass = pass && test("$a = b ? 2% | c 5%;"); // legal
|
|
pass = pass && test("$x = A B 5% C 10% | D;", 8, FALSE); // illegal 5%
|
|
pass = pass && test("$x = aa 45% | bb 5% cc;", 19, FALSE);// illegal cc
|
|
pass = pass && test("$x = (b 5%) (c 6%);"); // legal
|
|
pass = pass && test("$x = (b 5%) c 6%;", 13, FALSE); // illegal 6%
|
|
pass = pass && test("$x = b 5% (c 6%);", 9, FALSE); // illegal (c 6%)
|
|
pass = pass && test("$x = b 5% c 6%;", 9, FALSE); // illegal c 6%
|
|
pass = pass && test("$x = b 5%;"); // legal
|
|
pass = pass && test("$x = aa 45% | bb 5% cc;", 19, FALSE);// illegal cc
|
|
pass = pass && test("$x = a | b | c 4% | d 5%;"); // legal
|
|
pass = pass && test("$s = ' ' ? 50% abc;"); // legal
|
|
pass = pass && test("$s = a | c d | e f;"); // legal
|
|
pass = pass && test( "$z = q 0% | p 1% | r 100%;"); // legal How to check parsed tree??
|
|
|
|
pass = pass && test("$s = ' ' ? 50%;");
|
|
pass = pass && test("$relationList = '<' | '<<' | ';' | '<<<' | ',' | '=';");
|
|
pass = pass && test("$p1 = ($string $s '|' $s)? 25%;");
|
|
pass = pass && test("$p2 = (\\\\ $s $string $s)? 25%;");
|
|
pass = pass && test("$rel2 = $p1 $string $s $p2;");
|
|
pass = pass && test("$relation = $relationList $s ($rel1 | $rel2) $crlf;");
|
|
pass = pass && test("$command = $commandList $crlf;");
|
|
pass = pass && test("$reset = '&' $s ($beforeList $s)? 10% ($positionList 100% | $string 10%) $crlf;");
|
|
pass = pass && test("$mostRules = $command 1% | $reset 5% | $relation 25%;");
|
|
pass = pass && test("$root = $command{0,5} $reset $mostRules{1,20};");
|
|
|
|
const char collationBNF[] =
|
|
"$s = ' '? 50%;"
|
|
"$crlf = '\r\n';"
|
|
|
|
"$alternateOptions = non'-'ignorable | shifted;"
|
|
"$onoff = on | off;"
|
|
"$caseFirstOptions = off | upper | lower;"
|
|
"$strengthOptions = '1' | '2' | '3' | '4' | 'I';"
|
|
"$commandList = '['"
|
|
" ( alternate ' ' $alternateOptions"
|
|
" | backwards ' 2'"
|
|
" | normalization ' ' $onoff "
|
|
" | caseLevel ' ' $onoff "
|
|
" | hiraganaQ ' ' $onoff"
|
|
" | caseFirst ' ' $caseFirstOptions"
|
|
" | strength ' ' $strengthOptions"
|
|
" ) ']';"
|
|
"$command = $commandList $crlf;"
|
|
|
|
"$ignorableTypes = (tertiary | secondary | primary) ' ' ignorable;"
|
|
"$allTypes = variable | regular | implicit | trailing | $ignorableTypes;"
|
|
"$positionList = '[' (first | last) ' ' $allTypes ']';"
|
|
|
|
"$beforeList = '[before ' ('1' | '2' | '3') ']';"
|
|
|
|
"$relationList = ("
|
|
" '<'"
|
|
" | '<<'"
|
|
" | ';'"
|
|
" | '<<<'"
|
|
" | ','"
|
|
" | '='"
|
|
");"
|
|
"$string = $magic;"
|
|
"$rel1 = '[variable top]' $s;"
|
|
"$p1 = ($string $s '|' $s)? 25%;"
|
|
"$p2 = (\\\\ $s $string $s)? 25%;"
|
|
"$rel2 = $p1 $string $s $p2;"
|
|
"$relation = $relationList $s ($rel1 | $rel2) $crlf;"
|
|
|
|
"$reset = '&' $s ($beforeList $s)? 10% ($positionList 1% | $string 10%) $crlf;"
|
|
"$mostRules = $command 1% | $reset 5% | $relation 25%;"
|
|
"$root = $command{0,5} $reset $mostRules{1,20};"
|
|
;
|
|
|
|
pass = pass && test(collationBNF);
|
|
|
|
|
|
return pass;
|
|
}
|
|
|
|
static UBool TestMorph(){
|
|
srand((unsigned)time( NULL ));
|
|
|
|
Alternation * alt = new Alternation();
|
|
|
|
(*alt)
|
|
.append(new Literal("a")).append(new Literal("b")).append(new Literal("c"))
|
|
.append(new Literal("d")).append(new Literal("e")).append(new Literal("f"))
|
|
.append(new Literal("g")).append(new Literal("h")).append(new Literal("i"))
|
|
.append(new Literal("j")).append(new Literal("k")).append(new Literal("l"))
|
|
.append(new Literal("m")).append(new Literal("n")).append(new Literal("o"))
|
|
;
|
|
|
|
Repeat * rep = new Repeat( alt ,5,5 );
|
|
Morph m( *rep);
|
|
|
|
// DUMP_R(TestMorph,(*rep),20);
|
|
DUMP_R(TestMorph,m,100);
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
#endif
|
|
|
|
static UBool TestLanguageGenerator(){
|
|
//LanguageGenerator g;
|
|
//const char *const s = "$s = p 0% | q 1%;";
|
|
//g.parseBNF(s, "$s");
|
|
UBool pass;
|
|
//= strcmp("q", g.next()) == 0;
|
|
|
|
const char *const def =
|
|
//"$a = $b;"
|
|
//"$b = $c;"
|
|
//"$c = $t;"
|
|
//"$t = abc $z{1,2};"
|
|
//"$k = a | b | c | d | e | f | g ;"
|
|
//"$z = q 0% | p 1% | r 1%;"
|
|
"$x = a ? 0%;"
|
|
; // end of string
|
|
// const char * s = "abczz";
|
|
//
|
|
//
|
|
LanguageGenerator g;
|
|
pass = g.parseBNF(def, "$x",TRUE);
|
|
//// LanguageGenerator g(collationBNF, "$root", "$magic", new MagicNode());
|
|
//
|
|
if (pass != LanguageGenerator::OK) return FALSE;
|
|
|
|
DUMP_R(TestLanguageGenerator, g, 20);
|
|
return pass;
|
|
|
|
////UBool pass = strcmp(s,r) == 0;
|
|
|
|
//if (pass){
|
|
// printf("TestRandomLanguageGenerator passed.\n");
|
|
//} else {
|
|
// printf("TestRandomLanguageGenerator FAILED!!!\n");
|
|
//}
|
|
//return pass;
|
|
}
|
|
|
|
void TestWbnf(void){
|
|
srand((unsigned)time( NULL ));
|
|
|
|
//CALL(TestLiteral);
|
|
//CALL(TestSequence);
|
|
//CALL(TestSymbolTable);
|
|
//CALL(TestVariable);
|
|
|
|
//TestRepeat();
|
|
//TestAlternation();
|
|
//TestMorph();
|
|
|
|
//TestQuote();
|
|
//TestBuffer();
|
|
//TestWeightedRand();
|
|
|
|
//CALL(TestScanner);
|
|
//CALL(TestParser);
|
|
CALL(TestLanguageGenerator);
|
|
}
|
|
|