import ast
import re
import sys
import codecs

from json import JSONEncoder
from ast import *


# Is it Python 3?
is_python3 = hasattr(sys.version_info, 'major') and (sys.version_info.major == 3)


class AstEncoder(JSONEncoder):
    def default(self, o):
        if hasattr(o, '__dict__'):
            d = o.__dict__
            # workaround: decode strings if it's not Python3 code
            if not is_python3:
                for k in d:
                    if isinstance(d[k], str):
                        if k == 's':
                          d[k] = lines[d['start']:d['end']]
                        else:
                          d[k] = d[k].decode(enc)
            d['type'] = o.__class__.__name__
            return d
        else:
            return str(o)


enc = 'latin1'
lines = ''

def parse_dump(filename, output, end_mark):
    try:
        if is_python3:
            encoder = AstEncoder()
        else:
            encoder = AstEncoder(encoding=enc)

        tree = parse_file(filename)
        encoded = encoder.encode(tree)
        f = open(output, "w")
        f.write(encoded)
        f.close()
    finally:
        # write marker file to signal write end
        f = open(end_mark, "w")
        f.close()

def parse_json(filename):
    try:
        if is_python3:
            encoder = AstEncoder()
        else:
            encoder = AstEncoder(encoding=enc)

        tree = parse_file(filename)
        encoded = encoder.encode(tree)
        return encoded
    except:
        return ""


def parse_file(filename):
    global enc, lines
    enc, enc_len = detect_encoding(filename)
    f = codecs.open(filename, 'r', enc)
    lines = f.read()

    # remove BOM
    lines = re.sub(u'\ufeff', ' ', lines)

    # replace the encoding decl by spaces to fool python parser
    # otherwise you get 'encoding decl in unicode string' syntax error
    # print('enc:', enc, 'enc_len', enc_len)
    if enc_len > 0:
        lines = re.sub('#.*coding\s*[:=]\s*[\w\d\-]+',  '#' + ' ' * (enc_len-1), lines)

    f.close()
    return parse_string(lines, filename)


def parse_string(string, filename=None):
    tree = ast.parse(string)
    improve_ast(tree, string)
    if filename:
        tree.filename = filename
    return tree


# short function for experiments
def p(filename):
    parse_dump(filename, "json1", "end1")


def detect_encoding(path):
    fin = open(path, 'rb')
    prefix = str(fin.read(80))
    encs = re.findall('#.*coding\s*[:=]\s*([\w\d\-]+)', prefix)
    decl = re.findall('#.*coding\s*[:=]\s*[\w\d\-]+', prefix)

    if encs:
        enc1 = encs[0]
        enc_len = len(decl[0])
        try:
            info = codecs.lookup(enc1)
            # print('lookedup: ', info)
        except LookupError:
            # print('encoding not exist: ' + enc1)
            return 'latin1', enc_len
        return enc1, enc_len
    else:
        return 'latin1', -1


#-------------------------------------------------------------
#                   improvements to the AST
#-------------------------------------------------------------
def improve_ast(node, s):
    build_index_map(s)
    improve_node(node, s)


# build global table 'idxmap' for lineno <-> index oonversion
def build_index_map(s):
    global line_starts
    idx = 0
    line_starts = [0]
    while idx < len(s):
        if s[idx] == '\n':
            line_starts.append(idx + 1)
        idx += 1


# convert (line, col) to offset index
def map_idx(line, col):
    return line_starts[line - 1] + col


# convert offset index into (line, col)
def map_line_col(idx):
    line = 0
    for start in line_starts:
        if idx < start:
            break
        line += 1
    col = idx - line_starts[line - 1]
    return (line, col)


def improve_node(node, s):
    if isinstance(node, list):
        for n in node:
            improve_node(n, s)

    elif isinstance(node, AST):

        find_start(node, s)
        find_end(node, s)
        add_missing_names(node, s)

        for f in node_fields(node):
            improve_node(f, s)


def find_start(node, s):
    ret = None    # default value

    if hasattr(node, 'start'):
        ret = node.start

    elif isinstance(node, list):
        if node != []:
            ret = find_start(node[0], s)

    elif isinstance(node, Module):
        if node.body != []:
            ret = find_start(node.body[0], s)

    elif isinstance(node, BinOp):
        leftstart = find_start(node.left, s)
        if leftstart != None:
            ret = leftstart
        else:
            ret = map_idx(node.lineno, node.col_offset)

    elif hasattr(node, 'lineno'):
        if node.col_offset >= 0:
            ret = map_idx(node.lineno, node.col_offset)
        else:                           # special case for """ strings
            i = map_idx(node.lineno, node.col_offset)
            while i > 0 and i + 2 < len(s) and s[i:i + 3] != '"""' and s[i:i + 3] != "'''":
                i -= 1
            ret = i
    else:
        return None

    if ret == None and hasattr(node, 'lineno'):
        raise TypeError("got None for node that has lineno", node)

    if isinstance(node, AST) and ret != None:
        node.start = ret

    return ret


def find_end(node, s):
    the_end = None

    if hasattr(node, 'end'):
        return node.end

    elif isinstance(node, list):
        if node != []:
            the_end = find_end(node[-1], s)

    elif isinstance(node, Module):
        if node.body != []:
            the_end = find_end(node.body[-1], s)

    elif isinstance(node, Expr):
        the_end = find_end(node.value, s)

    elif isinstance(node, Str):
        i = find_start(node, s)
        while s[i] != '"' and s[i] != "'":
            i += 1

        if i + 2 < len(s) and s[i:i + 3] == '"""':
            q = '"""'
            i += 3
        elif i + 2 < len(s) and s[i:i + 3] == "'''":
            q = "'''"
            i += 3
        elif s[i] == '"':
            q = '"'
            i += 1
        elif s[i] == "'":
            q = "'"
            i += 1
        else:
            print("illegal quote:", i, s[i])
            q = ''

        if q != '':
            the_end = end_seq(s, q, i)

    elif isinstance(node, Name):
        the_end = find_start(node, s) + len(node.id)

    elif isinstance(node, Attribute):
        the_end = end_seq(s, node.attr, find_end(node.value, s))

    elif isinstance(node, FunctionDef):
        the_end = find_end(node.body, s)

    elif isinstance(node, Lambda):
        the_end = find_end(node.body, s)

    elif isinstance(node, ClassDef):
        the_end = find_end(node.body, s)

    # print will be a Call in Python 3
    elif not is_python3 and isinstance(node, Print):
        the_end = start_seq(s, '\n', find_start(node, s))

    elif isinstance(node, Call):
        start = find_end(node.func, s)
        if start != None:
            the_end = match_paren(s, '(', ')', start)

    elif isinstance(node, Yield):
        the_end = find_end(node.value, s)

    elif isinstance(node, Return):
        if node.value != None:
            the_end = find_end(node.value, s)
        else:
            the_end = find_start(node, s) + len('return')

    elif (isinstance(node, For) or
              isinstance(node, While) or
              isinstance(node, If) or
              isinstance(node, IfExp)):
        if node.orelse != []:
            the_end = find_end(node.orelse, s)
        else:
            the_end = find_end(node.body, s)

    elif isinstance(node, Assign) or isinstance(node, AugAssign):
        the_end = find_end(node.value, s)

    elif isinstance(node, BinOp):
        the_end = find_end(node.right, s)

    elif isinstance(node, BoolOp):
        the_end = find_end(node.values[-1], s)

    elif isinstance(node, Compare):
        the_end = find_end(node.comparators[-1], s)

    elif isinstance(node, UnaryOp):
        the_end = find_end(node.operand, s)

    elif isinstance(node, Num):
        the_end = find_start(node, s) + len(str(node.n))

    elif isinstance(node, List):
        the_end = match_paren(s, '[', ']', find_start(node, s));

    elif isinstance(node, Subscript):
        the_end = match_paren(s, '[', ']', find_start(node, s));

    elif isinstance(node, Tuple):
        if node.elts != []:
            the_end = find_end(node.elts[-1], s)

    elif isinstance(node, Dict):
        the_end = match_paren(s, '{', '}', find_start(node, s));

    elif ((not is_python3 and isinstance(node, TryExcept)) or
              (is_python3 and isinstance(node, Try))):
        if node.orelse != []:
            the_end = find_end(node.orelse, s)
        elif node.handlers != []:
            the_end = find_end(node.handlers, s)
        else:
            the_end = find_end(node.body, s)

    elif isinstance(node, ExceptHandler):
        the_end = find_end(node.body, s)

    elif isinstance(node, Pass):
        the_end = find_start(node, s) + len('pass')

    elif isinstance(node, Break):
        the_end = find_start(node, s) + len('break')

    elif isinstance(node, Continue):
        the_end = find_start(node, s) + len('continue')

    elif isinstance(node, Global):
        the_end = start_seq(s, '\n', find_start(node, s))

    elif isinstance(node, Import):
        the_end = find_start(node, s) + len('import')

    elif isinstance(node, ImportFrom):
        the_end = find_start(node, s) + len('from')

    else:   # can't determine node end, set to 3 chars after start
        start = find_start(node, s)
        if start != None:
            the_end = start + 3

    if isinstance(node, AST) and the_end != None:
        node.end = the_end

    return the_end


def add_missing_names(node, s):
    if hasattr(node, 'extra_attr'):
        return

    if isinstance(node, list):
        for n in node:
            add_missing_names(n, s)

    elif isinstance(node, ClassDef):
        head = find_start(node, s)
        start = s.find("class", head) + len("class")
        if start != None:
            node.name_node = str_to_name(s, start)
            node._fields += ('name_node',)

    elif isinstance(node, FunctionDef):
        # skip to "def" because it may contain decorators like @property
        head = find_start(node, s)
        start = s.find("def", head) + len("def")
        if start != None:
            node.name_node = str_to_name(s, start)
            node._fields += ('name_node',)

        # keyword_start = find_start(node, s)
        # node.keyword_node = str_to_name(s, keyword_start)
        # node._fields += ('keyword_node',)

        if node.args.vararg != None:
            if len(node.args.args) > 0:
                vstart = find_end(node.args.args[-1], s)
            else:
                vstart = find_end(node.name_node, s)
            if vstart != None:
                vname = str_to_name(s, vstart)
                node.vararg_name = vname
        else:
            node.vararg_name = None
        node._fields += ('vararg_name',)

        if node.args.kwarg != None:
            if len(node.args.args) > 0:
                kstart = find_end(node.args.args[-1], s)
            else:
                kstart = find_end(node.vararg_name, s)
            if kstart:
                kname = str_to_name(s, kstart)
                node.kwarg_name = kname
        else:
            node.kwarg_name = None
        node._fields += ('kwarg_name',)

    elif isinstance(node, Attribute):
        start = find_end(node.value, s)
        if start is not None:
            name = str_to_name(s, start)
            node.attr_name = name
            node._fields = ('value', 'attr_name')  # remove attr for node size accuracy

    elif isinstance(node, Compare):
        start = find_start(node, s)
        if start is not None:
            node.opsName = convert_ops(node.ops, s, start)
            node._fields += ('opsName',)

    elif (isinstance(node, BoolOp) or
              isinstance(node, BinOp) or
              isinstance(node, UnaryOp) or
              isinstance(node, AugAssign)):
        if hasattr(node, 'left'):
            start = find_end(node.left, s)
        else:
            start = find_start(node, s)
        if start is not None:
            ops = convert_ops([node.op], s, start)
        else:
            ops = []
        if ops != []:
            node.op_node = ops[0]
            node._fields += ('op_node',)

    elif isinstance(node, Num):
        if isinstance(node.n, int) or (not is_python3 and isinstance(node.n, long)):
            type = 'int'
            node.n = str(node.n)
        elif isinstance(node.n, float):
            type = 'float'
            node.n = str(node.n)
        elif isinstance(node.n, complex):
            type = 'complex'
            node.real = node.n.real
            node.imag = node.n.imag
            node._fields += ('real', 'imag')

        node.num_type = type
        node._fields += ('num_type',)

    node.extra_attr = True


#-------------------------------------------------------------
#              utilities used by improve AST functions
#-------------------------------------------------------------

# find a sequence in a string s, returning the start point
def start_seq(s, pat, start):
    try:
        return s.index(pat, start)
    except ValueError:
        return len(s)


# find a sequence in a string s, returning the end point
def end_seq(s, pat, start):
    try:
        return s.index(pat, start) + len(pat)
    except ValueError:
        return len(s)


# find matching close paren from start
def match_paren(s, open, close, start):
    while start < len(s) and s[start] != open:
        start += 1
    if start >= len(s):
        return len(s)

    left = 1
    i = start + 1
    while left > 0 and i < len(s):
        if s[i] == open:
            left += 1
        elif s[i] == close:
            left -= 1
        i += 1
    return i


# convert string to Name
def str_to_name(s, start):
    i = start;
    while i < len(s) and not is_alpha(s[i]):
        i += 1
    name_start = i

    ret = []
    while i < len(s) and is_alpha(s[i]):
        ret.append(s[i])
        i += 1
    name_end = i

    id1 = ''.join(ret)
    if id1 == '':
        return None
    else:
        name = Name(id1, None)
        name.start = name_start
        name.end = name_end
        name.lineno, name.col_offset = map_line_col(name_start)
        return name


def convert_ops(ops, s, start):
    syms = []
    for op in ops:
        if type(op) in ops_map:
            syms.append(ops_map[type(op)])
        else:
            print("[WARNING] operator %s is missing from ops_map, "
                  "please report the bug on GitHub" % op)

    i = start
    j = 0
    ret = []
    while i < len(s) and j < len(syms):
        oplen = len(syms[j])
        if s[i:i + oplen] == syms[j]:
            op_node = Name(syms[j], None)
            op_node.start = i
            op_node.end = i + oplen
            op_node.lineno, op_node.col_offset = map_line_col(i)
            ret.append(op_node)
            j += 1
            i = op_node.end
        else:
            i += 1
    return ret


# lookup table for operators for convert_ops
ops_map = {
    # compare:
    Eq: '==',
    NotEq: '!=',
    LtE: '<=',
    Lt: '<',
    GtE: '>=',
    Gt: '>',
    NotIn: 'not in',
    In: 'in',
    IsNot: 'is not',
    Is: 'is',

    # BoolOp
    Or: 'or',
    And: 'and',
    Not: 'not',
    Invert: '~',

    # bit operators
    BitOr: '|',
    BitAnd: '&',
    BitXor: '^',
    RShift: '>>',
    LShift: '<<',


    # BinOp
    Add: '+',
    Sub: '-',
    Mult: '*',
    Div: '/',
    FloorDiv: '//',
    Mod: '%',
    Pow: '**',

    # UnaryOp
    USub: '-',
    UAdd: '+',
}


# get list of fields from a node
def node_fields(node):
    ret = []
    for field in node._fields:
        if field != 'ctx' and hasattr(node, field):
            ret.append(getattr(node, field))
    return ret


# get full source text where the node is from
def node_source(node):
    if hasattr(node, 'node_source'):
        return node.node_source
    else:
        return None


# utility for getting exact source code part of the node
def src(node):
    return node.node_source[node.start: node.end]


def start(node):
    if hasattr(node, 'start'):
        return node.start
    else:
        return 0


def end(node):
    if hasattr(node, 'end'):
        return node.end
    else:
        return None


def is_alpha(c):
    return (c == '_'
            or ('0' <= c <= '9')
            or ('a' <= c <= 'z')
            or ('A' <= c <= 'Z'))


# p('/Users/yinwang/Code/django/tests/invalid_models/invalid_models/models.py')
# p('/Users/yinwang/Dropbox/prog/pysonar2/tests/test-unicode/test1.py')
# p('/Users/yinwang/Code/cpython/Lib/lib2to3/tests/data/bom.py')
# p('/Users/yinwang/Code/cpython/Lib/test/test_decimal.py')
# p('/Users/yinwang/Code/cpython/Lib/test/test_pep3131.py')
# p('/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/tarfile.py')
# p('/Users/yinwang/Code/cpython/Lib/lib2to3/tests/data/false_encoding.py')
# p('/System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5/test/test_marshal.py')
# p('/System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5/lib-tk/Tix.py')
#p('/home/liaoxinxi/trunk/src/bvs/login/test_system.py')
#p('/home/liaoxinxi/trunk/src/www/npai/systemforpa.py')
p('libssh2_login_test.py.bak')
#p('arg.py')