/*
* Javascript normalizer.
*
* Copyright (C) 2015 Cisco Systems, Inc. and/or its affiliates. All rights reserved.
* Copyright (C) 2008 Sourcefire, Inc.
*
* Authors: Török Edvin
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "clamav-config.h"
#endif
#include <stdio.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>
#include "clamav.h"
#include "cltypes.h"
#include "jsparse/lexglobal.h"
#include "hashtab.h"
#include "others.h"
#include "str.h"
#include "js-norm.h"
#include "jsparse/generated/operators.h"
#include "jsparse/generated/keywords.h"
#include "jsparse/textbuf.h"
/* ----------- tokenizer ---------------- */
enum tokenizer_state {
Initial,
MultilineComment,
SinglelineComment,
Number,
DoubleQString,
SingleQString,
Identifier,
Dummy
};
typedef struct scanner {
struct text_buffer buf;
const char *yytext;
size_t yylen;
const char *in;
size_t insize;
size_t pos;
size_t lastpos;
enum tokenizer_state state;
enum tokenizer_state last_state;
} *yyscan_t;
typedef int YY_BUFFER_STATE;
static int yylex( YYSTYPE *lvalp, yyscan_t );
static YY_BUFFER_STATE yy_scan_bytes( const char *, size_t, yyscan_t scanner );
static const char *yyget_text ( yyscan_t scanner );
static int yyget_leng ( yyscan_t scanner );
static int yylex_init ( yyscan_t * ptr_yy_globals ) ;
static int yylex_destroy ( yyscan_t yyscanner ) ;
/* ----------- tokenizer end ---------------- */
enum fsm_state {
Base,
InsideVar,
InsideInitializer,
WaitFunctionName,
WaitParameterList,
InsideFunctionDecl
};
struct scope {
struct cli_hashtable id_map;
struct scope *parent;/* hierarchy */
struct scope *nxt;/* all scopes kept in a list so we can easily free all of them */
enum fsm_state fsm_state;
int last_token;
unsigned int brackets;
unsigned int blocks;
};
struct tokens {
yystype *data;
size_t cnt;
size_t capacity;
};
/* state for the current JS file being parsed */
struct parser_state {
unsigned long var_uniq;
unsigned long syntax_errors;
struct scope *global;
struct scope *current;
struct scope *list;
yyscan_t scanner;
struct tokens tokens;
unsigned int rec;
};
static struct scope* scope_new(struct parser_state *state)
{
struct scope *parent = state->current;
struct scope *s = cli_calloc(1, sizeof(*s));
if(!s)
return NULL;
if(cli_hashtab_init(&s->id_map, 10) < 0) {
free(s);
return NULL;
}
s->parent = parent;
s->fsm_state = Base;
s->nxt = state->list;
state->list = s;
state->current = s;
return s;
}
static struct scope* scope_done(struct scope *s)
{
struct scope* parent = s->parent;
/* TODO: have a hashtab_destroy */
cli_hashtab_clear(&s->id_map);
free(s->id_map.htable);
free(s);
return parent;
}
/* transitions:
* Base --(VAR)--> InsideVar
* InsideVar --(Identifier)-->InsideInitializer
* InsideVar --(anything_else) --> POP (to Base)
* InsideInitializer --(COMMA)--> POP (to InsideVar)
* InsideInitializer | InsideVar --(SEMICOLON) --> POP (to Base)
* InsideInitializer --(BRACKET_OPEN) --> WaitBrClose
* InsideInitializer --(PAR_OPEN) --> WaitParClose
* WaitBrClose --(BRACKET_OPEN) --> increase depth
* WaitBrClose --(BRACKET_CLOSE) --> POP
* WaitParClose --(PAR_CLOSE) --> POP
* WaitParClose --(PAR_OPEN) --> increase depth
*/
/* Base --(VAR)--> PUSH, to InsideVar
* InsideVar --(Identifier)--> InsideInitializer
* InsideVar --(ELSE)--> POP, inc. syntax_errors
* InsideInitializer --(COMMA)--> POP (to InsideVar)
* --(BRACKET_OPEN)--> inc bracket_counter
* --(PAR_OPEN)--> inc par_counter
* --(BRACKET_CLOSE) --> dec bracket_counter
* --(PAR_CLOSE)--> dec par_counter
* --(VAR)--> PUSH, to InsideVar (if bracket_counter != 0 || par_counter != 0)
* --> POP, to InsideVar, inc. syntax_errors (if bracket_counter == 0 && par_counter == 0)
* POP only allowed if bracket_counter == 0 && par_counter == 0
*
* InsideInitializer acts differently, make it only a flag
* ....................
*
* Pushing, Poping is done when entering / exiting function scopes,
* tracking { and function ( is done by the function scope tracker too.
*
* we only need to track brackets.
*/
/*
* var x = document;
* x.writeln(...);
*
* ^we must not normalize member method names
*/
/*
* Variables are declared at function scope, and their initial value is
* undefined. At the point where the initializer is, and from there on the value
* is defined.
*
* { doesn't introduce a new variable scope, they are in function's scope too
*
* function foo() {
* alert(x); -> x exists, undefined
* var x=5;
* alert(x); -> x exists, =5
* }
*
* vs.
*
* function bar() {
* alert(x);//error, x not declared
* x=5;
* }
*
* vs.
*
* but we can declare variables without var, only valid if we use them after
* assigning.
*
* function foobar() {
* x=5;
* alert(x);//x is defined, value is 5
* }
*
* other examples:
* function foo2() {
* alert(x); -> x exists, undefined
* {
* var x=5; -> x equals to 5
* }
* alert(x); -> x is 5
* }
*
* function foo3() {
* var x=4; -> x exists, equals to 4
* alert(x); -> x exists, equals to 4
* {
* var x=5; -> x equals to 5
* }
* alert(x); -> x is 5
* }
*
* function bar3() {
* //same as foo3
* var x=4;
* alert(x);
* {
* x=5;
* }
* alert(x);
* }
*
*/
static const char* scope_declare(struct scope *s, const char *token, const size_t len, struct parser_state *state)
{
const struct cli_element *el = cli_hashtab_insert(&s->id_map, token, len, state->var_uniq++);
/* cli_hashtab_insert either finds an already existing entry, or allocates a
* new one, we return the allocated string */
return el ? el->key : NULL;
}
static const char* scope_use(struct scope *s, const char *token, const size_t len)
{
const struct cli_element *el = cli_hashtab_find(&s->id_map, token, len);
if(el) {
/* identifier already found in current scope,
* return here to avoid overwriting uniq id */
return el->key;
}
/* identifier not yet in current scope's hashtab, add with ID -1.
* Later if we find a declaration it will automatically assign a uniq ID
* to it. If not, we'll know that we have to push ID == -1 tokens to an
* outer scope.*/
el = cli_hashtab_insert(&s->id_map, token, len, -1);
return el ? el->key : NULL;
}
static long scope_lookup(struct scope *s, const char *token, const size_t len)
{
while(s) {
const struct cli_element *el = cli_hashtab_find(&s->id_map, token, len);
if(el && el->data != -1) {
return el->data;
}
/* not found in current scope, try in outer scope */
s = s->parent;
}
return -1;
}
static int tokens_ensure_capacity(struct tokens *tokens, size_t cap)
{
if(tokens->capacity < cap) {
yystype *data;
cap += 1024;
/* Keep old data if OOM */
data = cli_realloc(tokens->data, cap * sizeof(*tokens->data));
if(!data)
return CL_EMEM;
tokens->data = data;
tokens->capacity = cap;
}
return CL_SUCCESS;
}
static int add_token(struct parser_state *state, const yystype *token)
{
if(tokens_ensure_capacity(&state->tokens, state->tokens.cnt + 1))
return -1;
state->tokens.data[state->tokens.cnt++] = *token;
return 0;
}
struct buf {
size_t pos;
int outfd;
char buf[65536];
};
static inline int buf_outc(char c, struct buf *buf)
{
if(buf->pos >= sizeof(buf->buf)) {
if(write(buf->outfd, buf->buf, sizeof(buf->buf)) != sizeof(buf->buf))
return CL_EWRITE;
buf->pos = 0;
}
buf->buf[buf->pos++] = c;
return CL_SUCCESS;
}
static inline int buf_outs(const char *s, struct buf *buf)
{
const size_t buf_len = sizeof(buf->buf);
size_t i;
i = buf->pos;
while(*s) {
while(i < buf_len && *s) {
if(isspace(*s & 0xff))
buf->buf[i++] = ' ';
else
buf->buf[i++] = tolower((unsigned char)(*s));
++s;
}
if(i == buf_len) {
if(write(buf->outfd, buf->buf, buf_len) < 0)
return CL_EWRITE;
i = 0;
}
}
buf->pos = i;
return CL_SUCCESS;
}
static inline void output_space(char last, char current, struct buf *out)
{
if(isalnum(last) && isalnum(current))
buf_outc(' ', out);
}
/* return class of last character */
static char output_token(const yystype *token, struct scope *scope, struct buf *out, char lastchar)
{
char sbuf[128];
const char *s = TOKEN_GET(token, cstring);
/* TODO: use a local buffer, instead of FILE* */
switch(token->type) {
case TOK_StringLiteral:
output_space(lastchar,'"', out);
buf_outc('"', out);
if(s) {
buf_outs(s, out);
}
buf_outc('"', out);
return '\"';
case TOK_NumericInt:
output_space(lastchar,'0', out);
snprintf(sbuf, sizeof(sbuf), "%ld", TOKEN_GET(token, ival));
buf_outs(sbuf, out);
return '0';
case TOK_NumericFloat:
output_space(lastchar,'0', out);
snprintf(sbuf, sizeof(sbuf), "%g", TOKEN_GET(token, dval));
buf_outs(sbuf, out);
return '0';
case TOK_IDENTIFIER_NAME:
output_space(lastchar,'a', out);
if(s) {
long id = scope_lookup(scope, s, strlen(s));
if(id == -1) {
/* identifier not normalized */
buf_outs(s, out);
} else {
snprintf(sbuf, sizeof(sbuf), "n%03ld",id);
buf_outs(sbuf, out);
}
}
return 'a';
case TOK_FUNCTION:
output_space(lastchar,'a', out);
buf_outs("function",out);
return 'a';
default:
if(s) {
const size_t len = strlen(s);
output_space(lastchar,s[0], out);
buf_outs(s, out);
return len ? s[len-1] : '\0';
}
return '\0';
}
}
/*
* We can't delete the scope as soon as we see a }, because
* we still need the hashmap from it.
*
* If we would normalize all the identifiers, and output when a scope is closed,
* then it would be impossible to normalize calls to other functions.
*
* So we need to keep all scopes in memory, to do this instead of scope_done, we
* simply just set current = current->parent when a scope is closed.
* We keep a list of all scopes created in parser_state-> When we parsed
* everything, we output everything, and then delete all scopes.
*
* We also need to know where to switch scopes on the second pass, so for
* TOK_FUNCTION types we will use another pointer, that points to the scope
* (added to yystype's union).
*
* We lookup the identifier in the scope (using scope_lookup, it looks in parent
* scopes too), if ID is found then output (n%3d, Id),
* otherwise output the identifier as is.
*
* To make it easier to match sigs, we do a xfrm :
* 'function ID1 (..'. => 'n%3d = function (...'
*/
/*
* we'll add all identifier to the scope's map
* those that are not decl. will have initial ID -1
* if we later see a decl for it in same scope, it'll automatically get a
* correct ID.
*
* When parsing of local scope is done, we take any ID -1 identifiers,
* and push them up one level (careful not to overwrite existing IDs).
*
* it would be nice if the tokens would contain a link to the entry in the
* hashtab, a link that automatically gets updated when the element is moved
* (pushed up). This would prevent subsequent lookups in the map,
* when we want to output the tokens.
* There is no easy way to do that, so we just do another lookup
*
*/
/*
* This actually works, redefining foo:
* function foo() {
* var foo=5; alert(foo);
* }
* So we can't treat function names just as any other identifier?
* We can, because you can no longer call foo, if you redefined it as a var.
* So if we rename both foo-s with same name, it will have same behaviour.
*
* This means that a new scope should begin after function, and not after
* function ... (.
*/
static void scope_free_all(struct scope *p)
{
struct scope *nxt;
do {
nxt = p->nxt;
scope_done(p);
p = nxt;
} while(p);
}
size_t cli_strtokenize(char *buffer, const char delim, const size_t token_count, const char **tokens);
static int match_parameters(const yystype *tokens, const char ** param_names, size_t count)
{
size_t i,j=0;
if(tokens[0].type != TOK_PAR_OPEN)
return -1;
i=1;
while(count--) {
const char *token_val = TOKEN_GET(&tokens[i], cstring);
if(tokens[i].type != TOK_IDENTIFIER_NAME ||
!token_val ||
strcmp(token_val, param_names[j++]))
return -1;
++i;
if((count && tokens[i].type != TOK_COMMA)
|| (!count && tokens[i].type != TOK_PAR_CLOSE))
return -1;
++i;
}
return 0;
}
static const char *de_packer_3[] = {"p","a","c","k","e","r"};
static const char *de_packer_2[] = {"p","a","c","k","e","d"};
static inline char *textbuffer_done(yyscan_t scanner)
{
char *str = cli_realloc(scanner->buf.data, scanner->buf.pos);
if(!str) {
str = scanner->buf.data;
}
scanner->yytext = str;
scanner->yylen = scanner->buf.pos - 1;
memset(&scanner->buf, 0, sizeof(scanner->buf));
return str;
}
#define MODULE "JS-Norm: "
static void free_token(yystype *token)
{
if(token->vtype == vtype_string) {
free(token->val.string);
token->val.string = NULL;
}
}
static int replace_token_range(struct tokens *dst, size_t start, size_t end, const struct tokens *with)
{
const size_t len = with ? with->cnt : 0;
size_t i;
cli_dbgmsg(MODULE "Replacing tokens %lu - %lu with %lu tokens\n", (unsigned long)start,
(unsigned long)end, (unsigned long)len);
if(start >= dst->cnt || end > dst->cnt)
return -1;
for(i=start;i<end;i++) {
free_token(&dst->data[i]);
}
if(tokens_ensure_capacity(dst, dst->cnt - (end-start) + len))
return CL_EMEM;
memmove(&dst->data[start+len], &dst->data[end], (dst->cnt - end) * sizeof(dst->data[0]));
if(with && len > 0) {
memcpy(&dst->data[start], with->data, len * sizeof(dst->data[0]));
}
dst->cnt = dst->cnt - (end-start) + len;
return CL_SUCCESS;
}
static int append_tokens(struct tokens *dst, const struct tokens *src)
{
if(!dst || !src)
return CL_ENULLARG;
if(tokens_ensure_capacity(dst, dst->cnt + src->cnt))
return CL_EMEM;
cli_dbgmsg(MODULE "Appending %lu tokens\n", (unsigned long)(src->cnt));
memcpy(&dst->data[dst->cnt], src->data, src->cnt * sizeof(dst->data[0]));
dst->cnt += src->cnt;
return CL_SUCCESS;
}
static void decode_de(yystype *params[], struct text_buffer *txtbuf)
{
const char *p = TOKEN_GET(params[0], cstring);
const long a = TOKEN_GET(params[1], ival);
/*const char *c = params[2];*/
char *k = TOKEN_GET(params[3], string);
/*const char *r = params[5];*/
unsigned val=0;
unsigned nsplit = 0;
const char* o;
const char **tokens;
memset(txtbuf, 0, sizeof(*txtbuf));
if(!p || !k )
return;
for(o = k; *o; o++) if(*o == '|') nsplit++;
nsplit++;
tokens = malloc(sizeof(char*)*nsplit);
if(!tokens) {
return;
}
cli_strtokenize(k,'|',nsplit, tokens);
do {
while(*p && !isalnum(*p)) {
if(*p=='\\' && (p[1] == '\'' || p[1] == '\"'))
p++;
else
textbuffer_putc(txtbuf, *p++);
}
if(!*p) break;
val = 0;
o = p;
while(*p && isalnum(*p)) {
unsigned x;
unsigned char v = *p++;
/* TODO: use a table here */
if(v >= 'a') x = 10+v-'a';
else if(v >= 'A') x = 36+v-'A';
else x = v-'0';
val = val*a+x;
}
if(val >= nsplit || !tokens[val] || !tokens[val][0])
while(o!=p)
textbuffer_putc(txtbuf, *o++);
else textbuffer_append(txtbuf, tokens[val]);
} while (*p);
free(tokens);
textbuffer_append(txtbuf, "\0");
}
struct decode_result {
struct text_buffer txtbuf;
size_t pos_begin;
size_t pos_end;
unsigned append:1; /* 0: tokens are replaced with new token(s),
1: old tokens are deleted, new ones appended at the end */
};
static void handle_de(yystype *tokens, size_t start, const size_t cnt, const char *name, struct decode_result *res)
{
/* find function decl. end */
size_t i, nesting = 1, j;
yystype* parameters [6];
const size_t parameters_cnt = 6;
for(i=start;i < cnt; i++) {
if(tokens[i].type == TOK_FUNCTION) {
if(TOKEN_GET(&tokens[i], scope))
nesting++;
else
nesting--;
if(!nesting)
break;
}
}
if(nesting)
return;
memset(parameters, 0, sizeof(parameters));
if(name) {
/* find call to function */
for(;i+2 < cnt; i++) {
const char* token_val = TOKEN_GET(&tokens[i], cstring);
if(tokens[i].type == TOK_IDENTIFIER_NAME &&
token_val &&
!strcmp(name, token_val) &&
tokens[i+1].type == TOK_PAR_OPEN) {
i += 2;
for(j = 0;j < parameters_cnt && i < cnt;j++) {
parameters[j] = &tokens[i++];
if(j != parameters_cnt-1)
while (tokens[i].type != TOK_COMMA && i < cnt) i++;
else
while (tokens[i].type != TOK_PAR_CLOSE && i < cnt) i++;
i++;
}
if(j == parameters_cnt)
decode_de(parameters, &res->txtbuf);
}
}
} else {
while(i<cnt && tokens[i].type != TOK_PAR_OPEN) i++;
++i;
if(i >= cnt) return;
/* TODO: move this v to another func */
for(j = 0;j < parameters_cnt && i < cnt;j++) {
parameters[j] = &tokens[i++];
if(j != parameters_cnt-1)
while (tokens[i].type != TOK_COMMA && i < cnt) i++;
else
while (tokens[i].type != TOK_PAR_CLOSE && i < cnt) i++;
i++;
}
if(j == parameters_cnt)
decode_de(parameters, &res->txtbuf);
}
if(parameters[0] && parameters[parameters_cnt-1]) {
res->pos_begin = parameters[0] - tokens;
res->pos_end = parameters[parameters_cnt-1] - tokens + 1;
if(tokens[res->pos_end].type == TOK_BRACKET_OPEN &&
tokens[res->pos_end+1].type == TOK_BRACKET_CLOSE &&
tokens[res->pos_end+2].type == TOK_PAR_CLOSE)
res->pos_end += 3; /* {}) */
else
res->pos_end++; /* ) */
}
}
static int handle_unescape(struct tokens *tokens, size_t start)
{
if(tokens->data[start].type == TOK_StringLiteral) {
char *R;
struct tokens new_tokens;
yystype tok;
R = cli_unescape(TOKEN_GET(&tokens->data[start], cstring));
tok.type = TOK_StringLiteral;
TOKEN_SET(&tok, string, R);
new_tokens.capacity = new_tokens.cnt = 1;
new_tokens.data = &tok;
if(replace_token_range(tokens, start-2, start+2, &new_tokens) < 0)
return CL_EMEM;
}
return CL_SUCCESS;
}
/* scriptasylum dot com's JS encoder */
static void handle_df(const yystype *tokens, size_t start, struct decode_result *res)
{
char *str, *s1;
size_t len, s1_len, i;
unsigned char clast;
char *R;
if(tokens[start].type != TOK_StringLiteral)
return;
str = TOKEN_GET(&tokens[start], string);
if(!str)
return;
len = strlen(str);
clast = str[len-1] - '0';
str[len-1] = '\0';
s1 = cli_unescape(str);
s1_len = strlen(s1);
for(i=0;i<s1_len;i++) {
s1[i] -= clast;
}
R = cli_unescape(s1);
free(s1);
res->pos_begin = start-2;
res->pos_end = start+2;
res->txtbuf.data = R;
res->txtbuf.pos = strlen(R);
res->append = 1;
}
static void handle_eval(struct tokens *tokens, size_t start, struct decode_result *res)
{
res->txtbuf.data = TOKEN_GET(&tokens->data[start], string);
if(res->txtbuf.data && tokens->data[start+1].type == TOK_PAR_CLOSE) {
TOKEN_SET(&tokens->data[start], string, NULL);
res->txtbuf.pos = strlen(res->txtbuf.data);
res->pos_begin = start-2;
res->pos_end = start+2;
}
}
static void run_folders(struct tokens *tokens)
{
size_t i;
for(i = 0; i < tokens->cnt; i++) {
const char *cstring = TOKEN_GET(&tokens->data[i], cstring);
if(i+2 < tokens->cnt && tokens->data[i].type == TOK_IDENTIFIER_NAME &&
cstring &&
!strcmp("unescape", cstring) && tokens->data[i+1].type == TOK_PAR_OPEN) {
handle_unescape(tokens, i+2);
}
}
}
static inline int state_update_scope(struct parser_state *state, const yystype *token)
{
if(token->type == TOK_FUNCTION) {
struct scope *scope = TOKEN_GET(token, scope);
if(scope) {
state->current = scope;
}
else {
/* dummy token marking function end */
if(state->current->parent)
state->current = state->current->parent;
/* don't output this token, it is just a dummy marker */
return 0;
}
}
return 1;
}
static void run_decoders(struct parser_state *state)
{
size_t i;
const char* name;
struct tokens *tokens = &state->tokens;
for(i = 0; i < tokens->cnt; i++) {
const char *cstring = TOKEN_GET(&tokens->data[i], cstring);
struct decode_result res;
res.pos_begin = res.pos_end = 0;
res.append = 0;
if(tokens->data[i].type == TOK_FUNCTION && i+13 < tokens->cnt) {
name = NULL;
++i;
if(tokens->data[i].type == TOK_IDENTIFIER_NAME) {
cstring = TOKEN_GET(&tokens->data[i], cstring);
name = cstring;
++i;
}
if(match_parameters(&tokens->data[i], de_packer_3, sizeof(de_packer_3)/sizeof(de_packer_3[0])) != -1
|| match_parameters(&tokens->data[i], de_packer_2, sizeof(de_packer_2)/sizeof(de_packer_2[0])) != -1) {
/* find function decl. end */
handle_de(tokens->data, i, tokens->cnt, name, &res);
}
} else if(i+2 < tokens->cnt && tokens->data[i].type == TOK_IDENTIFIER_NAME &&
cstring &&
!strcmp("dF", cstring) && tokens->data[i+1].type == TOK_PAR_OPEN) {
/* TODO: also match signature of dF function (possibly
* declared using unescape */
handle_df(tokens->data, i+2, &res);
} else if(i+2 < tokens->cnt && tokens->data[i].type == TOK_IDENTIFIER_NAME &&
cstring &&
!strcmp("eval", cstring) && tokens->data[i+1].type == TOK_PAR_OPEN) {
handle_eval(tokens, i+2, &res);
}
if(res.pos_end > res.pos_begin) {
struct tokens parent_tokens;
if(res.pos_end < tokens->cnt && tokens->data[res.pos_end].type == TOK_SEMICOLON)
res.pos_end++;
parent_tokens = state->tokens;/* save current tokens */
/* initialize embedded context */
memset(&state->tokens, 0, sizeof(state->tokens));
if(++state->rec > 16)
cli_dbgmsg(MODULE "recursion limit reached\n");
else {
cli_js_process_buffer(state, res.txtbuf.data, res.txtbuf.pos);
--state->rec;
}
free(res.txtbuf.data);
/* state->tokens still refers to the embedded/nested context
* here */
if(!res.append) {
replace_token_range(&parent_tokens, res.pos_begin, res.pos_end, &state->tokens);
} else {
/* delete tokens */
replace_token_range(&parent_tokens, res.pos_begin, res.pos_end, NULL);
append_tokens(&parent_tokens, &state->tokens);
}
/* end of embedded context, restore tokens state */
free(state->tokens.data);
state->tokens = parent_tokens;
}
state_update_scope(state, &state->tokens.data[i]);
}
}
void cli_js_parse_done(struct parser_state* state)
{
struct tokens * tokens = &state->tokens;
size_t par_balance = 0, i;
char end = '\0';
YYSTYPE val;
cli_dbgmsg(MODULE "in cli_js_parse_done()\n");
/* close unfinished token */
switch (state->scanner->state) {
case DoubleQString:
end = '"';
break;
case SingleQString:
end = '\'';
break;
default: /* make gcc happy */
break;
}
if (end != '\0')
cli_js_process_buffer(state, &end, 1);
/* close remaining paranthesis */
for (i=0;i<tokens->cnt;i++) {
if (tokens->data[i].type == TOK_PAR_OPEN)
par_balance++;
else if (tokens->data[i].type == TOK_PAR_CLOSE && par_balance > 0)
par_balance--;
}
if (par_balance > 0) {
memset(&val, 0, sizeof(val));
val.type = TOK_PAR_CLOSE;
TOKEN_SET(&val, cstring, ")");
while (par_balance-- > 0) {
add_token(state, &val);
}
}
/* we had to close unfinished strings, paranthesis,
* so that the folders/decoders can run properly */
run_folders(&state->tokens);
run_decoders(state);
yylex_destroy(state->scanner);
state->scanner = NULL;
}
void cli_js_output(struct parser_state *state, const char *tempdir)
{
unsigned i;
struct buf buf;
char lastchar = '\0';
char filename[1024];
snprintf(filename, 1024, "%s"PATHSEP"javascript", tempdir);
buf.pos = 0;
buf.outfd = open(filename, O_CREAT | O_WRONLY, 0600);
if(buf.outfd < 0) {
cli_errmsg(MODULE "cannot open output file for writing: %s\n", filename);
return;
}
/* append to file */
if(lseek(buf.outfd, 0, SEEK_END) != 0) {
/* separate multiple scripts with \n */
buf_outc('\n', &buf);
}
buf_outs("<script>", &buf);
state->current = state->global;
for(i = 0; i < state->tokens.cnt; i++) {
if(state_update_scope(state, &state->tokens.data[i]))
lastchar = output_token(&state->tokens.data[i], state->current, &buf, lastchar);
}
/* add /script if not already there */
if(buf.pos < 9 || memcmp(buf.buf + buf.pos - 9, "</script>", 9))
buf_outs("</script>", &buf);
if(write(buf.outfd, buf.buf, buf.pos) < 0) {
cli_dbgmsg(MODULE "I/O error\n");
}
close(buf.outfd);
cli_dbgmsg(MODULE "dumped/appended normalized script to: %s\n",filename);
}
void cli_js_destroy(struct parser_state *state)
{
size_t i;
if(!state)
return;
scope_free_all(state->list);
for(i=0;i<state->tokens.cnt;i++) {
free_token(&state->tokens.data[i]);
}
free(state->tokens.data);
/* detect use after free */
if(state->scanner)
yylex_destroy(state->scanner);
memset(state, 0x55, sizeof(*state));
free(state);
cli_dbgmsg(MODULE "cli_js_destroy() done\n");
}
/* buffer is html-normlike "chunk", if original file is bigger than buffer,
* we rewind to a space, so we'll know that tokens won't be broken in half at
* the end of a buffer. All tokens except string-literals of course.
* So we can assume that after the buffer there is either a space, EOF, or a
* chunk of text not containing whitespace at all (for which we care only if its
* a stringliteral)*/
void cli_js_process_buffer(struct parser_state *state, const char *buf, size_t n)
{
struct scope* current = state->current;
YYSTYPE val;
int yv;
YY_BUFFER_STATE yyb;
if(!state->global) {
/* this state has either not been initialized,
* or cli_js_parse_done() was already called on it */
cli_warnmsg(MODULE "invalid state\n");
return;
}
yyb = yy_scan_bytes(buf, n, state->scanner);
memset(&val, 0, sizeof(val));
val.vtype = vtype_undefined;
/* on EOF yylex will return 0 */
while( (yv=yylex(&val, state->scanner)) != 0)
{
const char *text;
size_t leng;
val.type = yv;
switch(yv) {
case TOK_VAR:
current->fsm_state = InsideVar;
break;
case TOK_IDENTIFIER_NAME:
text = yyget_text(state->scanner);
leng = yyget_leng(state->scanner);
if(current->last_token == TOK_DOT) {
/* this is a member name, don't normalize
*/
TOKEN_SET(&val, string, cli_strdup(text));
val.type = TOK_UNNORM_IDENTIFIER;
} else {
switch(current->fsm_state) {
case WaitParameterList:
state->syntax_errors++;
/* fall through */
case Base:
case InsideInitializer:
TOKEN_SET(&val, cstring, scope_use(current, text, leng));
break;
case InsideVar:
case InsideFunctionDecl:
TOKEN_SET(&val, cstring, scope_declare(current, text, leng, state));
current->fsm_state = InsideInitializer;
current->brackets = 0;
break;
case WaitFunctionName:
TOKEN_SET(&val, cstring, scope_declare(current, text, leng, state));
current->fsm_state = WaitParameterList;
break;
}
}
break;
case TOK_PAR_OPEN:
switch(current->fsm_state) {
case WaitFunctionName:
/* fallthrough */
case WaitParameterList:
current->fsm_state = InsideFunctionDecl;
break;
default:
/* noop */
break;
}
break;
case TOK_PAR_CLOSE:
switch(current->fsm_state) {
case WaitFunctionName:
state->syntax_errors++;
break;
case WaitParameterList:
current->fsm_state = Base;
break;
default:
/* noop */
break;
}
break;
case TOK_CURLY_BRACE_OPEN:
switch(current->fsm_state) {
case WaitFunctionName:
/* fallthrough */
case WaitParameterList:
case InsideFunctionDecl:
/* in a syntactically correct
* file, we would already be in
* the Base state when we see a {
*/
current->fsm_state = Base;
/* fall-through */
case InsideVar:
case InsideInitializer:
state->syntax_errors++;
/* fall-through */
case Base:
default:
current->blocks++;
break;
}
break;
case TOK_CURLY_BRACE_CLOSE:
if(current->blocks > 0)
current->blocks--;
else
state->syntax_errors++;
if(!current->blocks) {
if(current->parent) {
/* add dummy FUNCTION token to
* mark function end */
TOKEN_SET(&val, cstring, "}");
add_token(state, &val);
TOKEN_SET(&val, scope, NULL);
val.type = TOK_FUNCTION;
state->current = current = current->parent;
} else{
/* extra } */
state->syntax_errors++;
}
}
break;
case TOK_BRACKET_OPEN:
current->brackets++;
break;
case TOK_BRACKET_CLOSE:
if(current->brackets > 0)
current->brackets--;
else
state->syntax_errors++;
break;
case TOK_COMMA:
if (current->fsm_state == InsideInitializer && current->brackets == 0 && current->blocks == 0) {
/* initializer ended only if we
* encountered a comma, and [] are
* balanced.
* This avoids switching state on:
* var x = [4,y,u];*/
current->fsm_state = InsideVar;
}
break;
case TOK_SEMICOLON:
if (current->brackets == 0 && current->blocks == 0) {
/* avoid switching state on unbalanced []:
* var x = [test;testi]; */
current->fsm_state = Base;
}
break;
case TOK_FUNCTION:
current = scope_new(state);
current->fsm_state = WaitFunctionName;
TOKEN_SET(&val, scope, state->current);
break;
case TOK_StringLiteral:
if(state->tokens.cnt > 1 && state->tokens.data[state->tokens.cnt-1].type == TOK_PLUS) {
/* see if can fold */
yystype *prev_string = &state->tokens.data[state->tokens.cnt-2];
if(prev_string->type == TOK_StringLiteral) {
char *str = TOKEN_GET(prev_string, string);
size_t str_len = strlen(str);
text = yyget_text(state->scanner);
leng = yyget_leng(state->scanner);
/* delete TOK_PLUS */
free_token(&state->tokens.data[--state->tokens.cnt]);
str = cli_realloc(str, str_len + leng + 1);
if (!str)
break;
strncpy(str+str_len, text, leng);
str[str_len + leng] = '\0';
TOKEN_SET(prev_string, string, str);
free(val.val.string);
memset(&val, 0, sizeof(val));
val.vtype = vtype_undefined;
continue;
}
}
break;
}
if(val.vtype == vtype_undefined) {
text = yyget_text(state->scanner);
TOKEN_SET(&val, string, cli_strdup(text));
abort();
}
add_token(state, &val);
current->last_token = yv;
memset(&val, 0, sizeof(val));
val.vtype = vtype_undefined;
}
}
struct parser_state *cli_js_init(void)
{
struct parser_state *state = cli_calloc(1, sizeof(*state));
if(!state)
return NULL;
if(!scope_new(state)) {
free(state);
return NULL;
}
state->global = state->current;
if(yylex_init(&state->scanner)) {
scope_done(state->global);
free(state);
return NULL;
}
cli_dbgmsg(MODULE "cli_js_init() done\n");
return state;
}
/*-------------- tokenizer ---------------------*/
enum char_class {
Whitespace,
Slash,
Operator,
DQuote,
SQuote,
Digit,
IdStart,
BracketOpen = TOK_BRACKET_OPEN,
BracketClose = TOK_BRACKET_CLOSE,
Comma = TOK_COMMA,
CurlyOpen = TOK_CURLY_BRACE_OPEN,
CurlyClose = TOK_CURLY_BRACE_CLOSE,
ParOpen = TOK_PAR_OPEN,
ParClose = TOK_PAR_CLOSE,
Dot = TOK_DOT,
SemiColon = TOK_SEMICOLON,
Nop
};
#define SL Slash
#define DG Digit
#define DQ DQuote
#define SQ SQuote
#define ID IdStart
#define OP Operator
#define WS Whitespace
#define BO BracketOpen
#define BC BracketClose
#define CM Comma
#define CO CurlyOpen
#define CC CurlyClose
#define PO ParOpen
#define PC ParClose
#define DT Dot
#define SC SemiColon
#define NA Nop
static const enum char_class ctype[256] = {
NA, NA, NA, NA, NA, NA, NA, NA, NA, WS, WS, WS, NA, WS, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
WS, OP, DQ, NA, ID, OP, OP, SQ, PO, PC, OP, OP, CM, OP, DT, SL,
DG, DG, DG, DG, DG, DG, DG, DG, DG, DG, OP, SC, OP, OP, OP, OP,
NA, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID,
ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, BO, ID, BC, OP, ID,
NA, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID,
ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, CO, OP, CC, OP, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA
};
static const enum char_class id_ctype[256] = {
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, ID, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, NA, NA, NA, NA, NA, NA,
NA, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID,
ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, NA, OP, NA, NA, ID,
NA, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID,
ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, ID, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
};
#define CASE_SPECIAL_CHAR(C, S) case C: TOKEN_SET(lvalp, cstring, (S)); return cClass;
#define BUF_KEEP_SIZE 32768
static void textbuf_clean(struct text_buffer *buf)
{
if(buf->capacity > BUF_KEEP_SIZE) {
char *data= cli_realloc(buf->data, BUF_KEEP_SIZE);
if (data)
buf->data = data;
buf->capacity = BUF_KEEP_SIZE;
}
buf->pos = 0;
}
static inline int parseString(YYSTYPE *lvalp, yyscan_t scanner, const char q,
enum tokenizer_state tostate)
{
size_t len;
/* look for " terminating the string */
const char *start = &scanner->in[scanner->pos], *end = start;
do {
const size_t siz = &scanner->in[scanner->insize] - end;
end = memchr(end, q, siz);
if(end && end > start && end[-1] == '\\') {
++end;
continue;
}
break;
} while (1);
if(end && end >= start)
len = end - start;
else
len = scanner->insize - scanner->pos;
cli_textbuffer_append_normalize(&scanner->buf, start, len);
if(end) {
char *str;
/* skip over end quote */
scanner->pos += len + 1;
textbuffer_putc(&scanner->buf, '\0');
str = textbuffer_done(scanner);
if (str) {
TOKEN_SET(lvalp, string, str);
} else {
TOKEN_SET(lvalp, cstring, "");
}
scanner->state = Initial;
assert(lvalp->val.string);
return TOK_StringLiteral;
} else {
scanner->pos += len;
/* unfinished string */
scanner->state = tostate;
return 0;
}
}
static inline int parseDQString(YYSTYPE *lvalp, yyscan_t scanner)
{
return parseString(lvalp, scanner, '"', DoubleQString);
}
static inline int parseSQString(YYSTYPE *lvalp, yyscan_t scanner)
{
return parseString(lvalp, scanner, '\'', SingleQString);
}
static inline int parseNumber(YYSTYPE *lvalp, yyscan_t scanner)
{
const unsigned char *in = (const unsigned char*)scanner->in;
int is_float = 0;
while(scanner->pos < scanner->insize) {
unsigned char c = in[scanner->pos++];
if(isdigit(c)) {
textbuffer_putc(&scanner->buf, c);
continue;
}
if(c =='.' && !is_float) {
is_float = 1;
textbuffer_putc(&scanner->buf, '.');
continue;
}
if((c=='e' || c=='E') && is_float) {
textbuffer_putc(&scanner->buf, c);
if(scanner->pos < scanner->insize) {
c = in[scanner->pos++];
if(c == '+' || c == '-' || isdigit(c)) {
textbuffer_putc(&scanner->buf, c);
continue;
}
}
}
scanner->pos--;
textbuffer_putc(&scanner->buf, '\0');
scanner->state = Initial;
if (!scanner->buf.data)
return 0;
if(is_float) {
TOKEN_SET(lvalp, dval, atof(scanner->buf.data));
return TOK_NumericFloat;
} else {
TOKEN_SET(lvalp, ival, atoi(scanner->buf.data));
return TOK_NumericInt;
}
}
scanner->state = Number;
return 0;
}
static inline int parseId(YYSTYPE *lvalp, yyscan_t scanner)
{
const struct keyword *kw;
const unsigned char *in = (const unsigned char*)scanner->in;
scanner->state = Initial;
while(scanner->pos < scanner->insize) {
unsigned char c = in[scanner->pos++];
enum char_class cClass = id_ctype[c];
switch(cClass) {
case IdStart:
textbuffer_putc(&scanner->buf, c);
break;
case Operator:
/* the table contains OP only for \ */
assert(c == '\\');
if(scanner->pos < scanner->insize &&
in[scanner->pos++] == 'u') {
textbuffer_putc(&scanner->buf, c);
break;
}
if(scanner->pos == scanner->insize) {
scanner->pos++;
}
/* else fallthrough */
default:
/* character is no longer part of identifier */
scanner->state = Initial;
textbuffer_putc(&scanner->buf, '\0');
scanner->pos--;
kw = in_word_set(scanner->buf.data, scanner->buf.pos-1);
if(kw) {
/* we got a keyword */
TOKEN_SET(lvalp, cstring, kw->name);
return kw->val;
}
/* it is not a keyword, just an identifier */
TOKEN_SET(lvalp, cstring, NULL);
return TOK_IDENTIFIER_NAME;
}
}
scanner->state = Identifier;
return 0;
}
static int parseOperator(YYSTYPE *lvalp, yyscan_t scanner)
{
size_t len = MIN(5, scanner->insize - scanner->pos);
while(len) {
const struct operator *kw = in_op_set(&scanner->in[scanner->pos], len);
if(kw) {
TOKEN_SET(lvalp, cstring, kw->name);
scanner->pos += len;
return kw->val;
}
len--;
}
/* never reached */
assert(0);
scanner->pos++;
TOKEN_SET(lvalp, cstring, NULL);
return TOK_ERROR;
}
static int yylex_init(yyscan_t *scanner)
{
*scanner = cli_calloc(1, sizeof(**scanner));
return *scanner ? 0 : -1;
}
static int yylex_destroy(yyscan_t scanner)
{
free(scanner->buf.data);
free(scanner);
return 0;
}
static int yy_scan_bytes(const char *p, size_t len, yyscan_t scanner)
{
scanner->in = p;
scanner->insize = len;
scanner->pos = 0;
scanner->lastpos = -1;
scanner->last_state = Dummy;
return 0;
}
static const char *yyget_text(yyscan_t scanner)
{
return scanner->yytext ? scanner->yytext : scanner->buf.data;
}
static int yyget_leng(yyscan_t scanner)
{
/* we have a \0 too */
return scanner->yylen ? scanner->yylen: (scanner->buf.pos > 0 ? scanner->buf.pos - 1 : 0);
}
static int yylex(YYSTYPE *lvalp, yyscan_t scanner)
{
const size_t len = scanner->insize;
const unsigned char *in = (const unsigned char*)scanner->in;
unsigned char lookahead;
enum char_class cClass;
scanner->yytext = NULL;
scanner->yylen = 0;
if(scanner->pos == scanner->lastpos) {
if(scanner->last_state == scanner->state) {
cli_dbgmsg(MODULE "infloop detected, skipping character\n");
scanner->pos++;
}
/* its not necesarely an infloop if it changed
* state, and it shouldn't infloop between states */
}
scanner->lastpos = scanner->pos;
scanner->last_state = scanner->state;
while(scanner->pos < scanner->insize) {
switch(scanner->state) {
case Initial:
textbuf_clean(&scanner->buf);
cClass = ctype[in[scanner->pos++]];
switch(cClass) {
case Whitespace:
/* eat whitespace */
continue;
case Slash:
if(scanner->pos < len) {
lookahead = in[scanner->pos];
switch(lookahead) {
case '*':
scanner->state = MultilineComment;
scanner->pos++;
continue;
case '/':
scanner->state = SinglelineComment;
scanner->pos++;
continue;
}
}
--scanner->pos;
return parseOperator(lvalp, scanner);
case Operator:
--scanner->pos;
return parseOperator(lvalp, scanner);
case DQuote:
return parseDQString(lvalp, scanner);
case SQuote:
return parseSQString(lvalp, scanner);
case Digit:
--scanner->pos;
return parseNumber(lvalp, scanner);
case IdStart:
--scanner->pos;
return parseId(lvalp,scanner);
CASE_SPECIAL_CHAR(BracketOpen, "[");
CASE_SPECIAL_CHAR(BracketClose, "]");
CASE_SPECIAL_CHAR(Comma, ",");
CASE_SPECIAL_CHAR(CurlyOpen, "{");
CASE_SPECIAL_CHAR(CurlyClose, "}");
CASE_SPECIAL_CHAR(ParOpen, "(");
CASE_SPECIAL_CHAR(ParClose, ")");
CASE_SPECIAL_CHAR(Dot, ".");
CASE_SPECIAL_CHAR(SemiColon, ";");
case Nop:
continue;
}
break;
case DoubleQString:
return parseString(lvalp, scanner, '"', DoubleQString);
case SingleQString:
return parseString(lvalp, scanner, '\'', SingleQString);
case Identifier:
return parseId(lvalp, scanner);
case MultilineComment:
while(scanner->pos+1 < scanner->insize) {
if(in[scanner->pos] == '*' && in[scanner->pos+1] == '/') {
scanner->state = Initial;
scanner->pos++;
break;
}
scanner->pos++;
}
scanner->pos++;
break;
case Number:
return parseNumber(lvalp, scanner);
case SinglelineComment:
while(scanner->pos < scanner->insize) {
/* htmlnorm converts \n to space, so
* stop on space too */
if(in[scanner->pos] == '\n' || in[scanner->pos] == ' ')
break;
scanner->pos++;
}
scanner->state = Initial;
break;
default:
assert(0 && "Not reached");
}
}
return 0;
}