Andrew Dalke's writings

i

import pybel

for mol in pybel.readfile("sdf", "benzodiazepine.sdf.gz"):
    print mol.OBMol.NumHvyAtoms()

;)
>>> import ast

>>> ast.parse("for i in range(10): print i")
<_ast.Module object at 0x1004d06d0>
>>> 

>>> ast.dump(ast.parse("for i in range(10): print i"))
"Module(body=[For(target=Name(id='i', ctx=Store()),
   iter=Call(func=Name(id='range', ctx=Load()),
   args=[Num(n=10)], keywords=[], starargs=None,
   kwargs=None), body=[Print(dest=None,
   values=[Name(id='i', ctx=Load())], nl=True)],
   orelse=[])])"

>>> ast.dump(ast.parse("for i in range(10): print i"), True, True)
"Module(body=[For(target=Name(id='i', ctx=Store(),
   lineno=1, col_offset=4), iter=Call(func=Name(
   id='range', ctx=Load(), lineno=1, col_offset=9),
   args=[Num(n=10, lineno=1, col_offset=15)],
   keywords=[], starargs=None, kwargs=None,
   lineno=1, col_offset=9), body=[Print(dest=None,
   values=[Name(id='i', ctx=Load(), lineno=1, col_offset=26)],
   nl=True, lineno=1, col_offset=20)], orelse=[], lineno=1,
   col_offset=0)])"

>>> from ast import *
>>> tree = Module([Print(None, [Str("PyCon2010!")], True)])
>>> tree.lineno = 1
>>> tree.col_offset = 1
>>> fix_missing_locations(tree)
<_ast.Module object at 0x1004dff50>
>>> tree = fix_missing_locations(tree)
>>> compile(tree, "<TiP>", "exec")
<code object <module> at 0x1004d38a0, file "<TiP>", line 1>
>>> exec compile(tree, "<TiP>", "exec")
PyCon2010!
>>> 

try:
  raise TypeError("blah: %d" % "I said 'PyCon2010'!")
except TypeError:
  pass

try:
  raise TypeError(
         check_mod("blah: %d", "I said 'PyCon2010'!"))
except TypeError:
  pass

def check_mod(left, right):
    try:
        return left % right
    except Exception, err:
        print "Could not interpolate: %s" % (err,)
        traceback.print_stack()
        raise

from ast import *

class RewriteInterpolation(NodeTransformer):
    def visit_BinOp(self, node):
        if isinstance(node.op, Mod):
            new_node = Call(func=Name(id='check_string', ctx=Load()),
                            args=[node.left, node.right,
                                  Num(n=node.lineno),
                                  Num(n=node.col_offset)],
                            keywords = [], starargs=None, kwargs=None
                            )
            copy_location(new_node, node)
            fix_missing_locations(new_node)
            return new_node
        return node

code = open(filename).read()
tree = parse(code, filename)
tree = RewriteInterpolation().visit(tree)

        try:
            linenum = '%d' % linenum
            id = ' id="%s%s"' % (self._prefix[side],linenum)
        except TypeError:
            # handle blank lines where linenum is '>' or ''
            id = ''

x = 1

from ast_report import register_module

ast_enter, ast_leave, ast_reached = \
   register_module('spam/testing.py',
      {0: (1, 0)}, {} )   #  unique identifer -> (lineno, col_offset)

if 1:
  ast_enter[0] += 1
  x = 3
  ast_leave[0] += 1

# ast_compileall.py
import __builtin__
from ast import *
import compileall
import sys
import traceback
import itertools

class RewriteInterpolation(NodeTransformer):
    def __init__(self, filename):
        self.filename = filename
        self.enter_linenos = {}  # id -> (lineno, col_offset)
        self.reach_linenos = {}  # id -> (lineno, col_offset)
        self.counter = itertools.count()
    def visit_Module(self, module_node):
        # Need to import and call ast_report.register_module().
        # These must occur after the "from __future__ import ..." statements.
        # Find where I can insert them.
        body_future = []
        body_rest = []
        for node in module_node.body:
            node = self.visit(node)
            if (not body_rest and isinstance(node, ImportFrom) and
                node.module == "__future__"):
                body_future.append(node)
            else:
                body_rest.append(node)

        # It's easier to let Python convert the code to an AST
        import_line = parse("from ast_report import register_module, check_string").body[0]
        print ("ast_enter, ast_leave, ast_reached = register_module(%r, %r, %r)" %
               (self.filename, self.enter_linenos, self.reach_linenos))
        register_line = parse(
            "ast_enter, ast_leave, ast_reached = register_module(%r, %r, %r)" %
            (self.filename, self.enter_linenos, self.reach_linenos)).body[0]

        # Assign a reasonable seeming line number.
        lineno = 1
        if body_future:
            lineno = body_future[0].lineno
        for new_node in (import_line, register_line):
            new_node.col_offset = 1
            new_node.lineno = lineno

        new_body = body_future + [import_line, register_line] + body_rest
        return Module(body=new_body)

    # These are statements which should have an enter and leave
    # (In retrospect, this isn't always true, eg, for 'if')
    def track_enter_leave_lineno(self, node):
        node = self.generic_visit(node)
        id = next(self.counter)
        enter = parse("ast_enter[%d] += 1" % id).body[0]
        leave = parse("ast_leave[%d] += 1" % id).body[0]
        self.enter_linenos[id] = (node.lineno, node.col_offset)
        for new_node in (enter, leave):
            copy_location(new_node, node)

        # This is the code for "if 1: ..."
        n = Num(n=1)
        copy_location(n, node)
        if_node = If(test=n, body=[enter, node, leave], orelse=[])
        copy_location(if_node, node)
        return if_node

    visit_FunctionDef = track_enter_leave_lineno
    visit_ClassDef = track_enter_leave_lineno
    visit_Assign = track_enter_leave_lineno
    visit_AugAssign = track_enter_leave_lineno
    visit_Delete = track_enter_leave_lineno
    visit_Print = track_enter_leave_lineno
    visit_For = track_enter_leave_lineno
    visit_While = track_enter_leave_lineno
    visit_If = track_enter_leave_lineno
    visit_With = track_enter_leave_lineno
    visit_TryExcept = track_enter_leave_lineno
    visit_TryFinally = track_enter_leave_lineno
    visit_Assert = track_enter_leave_lineno
    visit_Import = track_enter_leave_lineno
    visit_ImportFrom = track_enter_leave_lineno
    visit_Exec = track_enter_leave_lineno
    #Global
    visit_Expr = track_enter_leave_lineno
    visit_Pass = track_enter_leave_lineno

    # These statements can be reached, but they change
    # control flow and are never exited.
    def track_reached_lineno(self, node):
        node = self.generic_visit(node)
        id = next(self.counter)
        reach = parse("ast_reached[%d] += 1" % id).body[0]
        self.reach_linenos[id] = (node.lineno, node.col_offset)
        copy_location(reach, node)

        n = Num(n=1)
        copy_location(n, node)
        if_node = If(test=n, body=[reach, node], orelse=[])
        copy_location(if_node, node)
        return if_node

    visit_Return = track_reached_lineno
    visit_Raise = track_reached_lineno
    visit_Break = track_reached_lineno
    visit_Continue = track_reached_lineno

    # Some code to instrument the run-time and check for '%' failures.
    def visit_BinOp(self, node):
        if isinstance(node.op, Mod):
            new_node = Call(func=Name(id='check_string', ctx=Load()),
                            args=[node.left, node.right,
                                  Num(n=node.lineno),
                                  Num(n=node.col_offset)],
                            keywords = [], starargs=None, kwargs=None
                            )
            copy_location(new_node, node)
            fix_missing_locations(new_node)
            return new_node
        return node

old_compile = __builtin__.compile

def compile(source, filename, mode, flags=0): # skipping a few parameters
    # My rewrite code uses ast.parse, which ends up calling this
    # function with this argument, so pass it back to the real compile.
    if flags == PyCF_ONLY_AST:
        return old_compile(source, filename, mode, flags)
    assert mode == "exec"
    #traceback.print_stack()
    code = open(filename).read()
    tree = parse(code, filename)
    tree = RewriteInterpolation(filename).visit(tree)
    code = old_compile(tree, filename, "exec")
    return code

# Ugly hack so I can force compileall to use my compile function.
__builtin__.compile = compile

exit_status = int(not compileall.main())
sys.exit(exit_status)

def main():

  def f(x):
    if x > 0:
      return x*x
    1/0

  for i in range(4, 9):
    if f(i) < 0: x=9
    if i == 8:
       continue
       print "Here"
    if i == 10:
       continue

  try:
      raise TypeError("Hi! %d" % "sdfa")
  except TypeError:
      pass

main()

python ast_compileall.py spam

# ast_report.py
from collections import defaultdict
import traceback
import atexit
import linecache

loaded_modules = []

class FileInfo(object):
    def __init__(self, filename, enter_linenos, reach_linenos):
        self.filename = filename
        self.enter_linenos = enter_linenos
        self.reach_linenos = reach_linenos
        self.ast_enter = defaultdict(int)
        self.ast_leave = defaultdict(int)
        self.ast_reach = defaultdict(int)

def register_module(filename, enter_linenos, reach_linenos):
    #print filename, enter_linenos, reach_linenos
    info = FileInfo(filename, enter_linenos, reach_linenos)
    loaded_modules.append(info)
    return info.ast_enter, info.ast_leave, info.ast_reach

def check_string(left, right, lineno, col_offset):
    if not isinstance(left, basestring):
        return left % right
    try:
        return left % right
    except Exception, err:
        print "Could not interpolate: %s" % (err,)
        traceback.print_stack()
        raise

# Basic coverage report
def report_coverage():
    for fileinfo in loaded_modules:
        # This will contain a list of all results as a 3-ple of
        #   lineno, col_offset, "text message"
        report = []
        # These should have both 'enter' and 'leave' counts.
        for id, (lineno, col_offset) in fileinfo.enter_linenos.items():
            if id not in fileinfo.ast_enter:
                report.append( (lineno, col_offset, "not entered") )
            elif id not in fileinfo.ast_leave:
                report.append( (lineno, col_offset, "enter %d but never left" %
                                fileinfo.ast_enter[id]) )
            else:
                delta = fileinfo.ast_leave[id] - fileinfo.ast_enter[id]
                report.append( (lineno, col_offset, "enter %d leave %d (diff %d)" %
                                (fileinfo.ast_enter[id], fileinfo.ast_leave[id], delta)) )

        # These only need to be 'reach'ed
        for id, (lineno, col_offset) in fileinfo.reach_linenos.items():
            if id not in fileinfo.ast_reach:
                report.append( (lineno, col_offset, "not reached") )
            else:
                report.append( (lineno, col_offset, "reach %d" % (fileinfo.ast_reach[id],)) )

        # sort by line number, breaking ties by column offset
        report.sort()

        print "Coverage results for file", fileinfo.filename
        for lineno, col_offset, msg in report:
            print "%d:%d %s" % (lineno, col_offset+1, msg)
            print linecache.getline(fileinfo.filename, lineno).rstrip()

# Dump the coverage results when Python exist.
atexit.register(report_coverage)

% cd spam/
% python -c 'import testing'
Could not interpolate: %d format: a number is required, not str
  File "<string>", line 1, in <module>
  File "spam/testing.py", line 21, in <module>
    main()
  File "spam/testing.py", line 785, in main
  File "ast_report.py", line 30, in check_string
    traceback.print_stack()
Coverage results for file spam/testing.py
1:1 enter 1 leave 1 (diff 0)
def main():
3:3 enter 1 leave 1 (diff 0)
  def f(x):
4:5 enter 5 but never left
    if x > 0:
5:7 reach 5
      return x*x
6:5 not entered
    1/0
8:3 enter 1 leave 1 (diff 0)
  for i in range(4, 9):
9:5 enter 5 leave 5 (diff 0)
    if f(i) < 0: x=9
9:18 not entered
    if f(i) < 0: x=9
10:5 enter 5 leave 4 (diff -1)
    if i == 8:
11:8 reach 1
       continue
12:8 not entered
       print "Here"
13:5 enter 4 leave 4 (diff 0)
    if i == 10:
14:8 not reached
       continue
16:3 enter 1 leave 1 (diff 0)
  try:
17:7 reach 1
      raise TypeError("Hi! %d" % "sdfa")
19:7 enter 1 leave 1 (diff 0)
      pass
21:1 enter 1 leave 1 (diff 0)
main()

x = arg or default_arg or die(_("missing arg"))

 #FPS1
 #num_bits=256
 #software=RDKit/2009Q3_1
 #params=RDKit-Fingerprint/1 minPath=1 maxPath=7 fpSize=256 nBitsPerHash=4 useHs=True
 #source=/Users/dalke/databases/Compound_00000001_00025000.sdf.gz
 #date=2010-01-27T02:22:26
 fffeffbfb7fffedff7beefdbddf7ffffabff76cf6df7fcf6f7fffebf7d7ffd6f 1
 fffeffbfb7fffedff7beefdbddf7ffffabff76cf6df7fcf6f7fffebf7d7ffd6f 2
 ffffbfdfffffffffbfeffffffffffffffffffffffff77efffffffebfffffffef 3
 00c02010002610000080800041100002084000440d100000c055048801224400 4

0x89 0x46 0x50 0x46 0x0D 0x0A 0x1A 0x0A
0x46 0x50 0x46
   Title            Short (one line) title or caption for image
   Author           Name of image's creator
   Description      Description of image (possibly long)
   Copyright        Copyright notice
   Creation Time    Time of original image creation
   Software         Software used to create the image
   Disclaimer       Legal disclaimer
   Warning          Warning of nature of content
   Source           Device used to create the image
   Comment          Miscellaneous comment; conversion from
                    GIF comment

[ integer number of bits per fingerprint ]
[ integer number of bytes per fingerprint (including alignment) ]
[ integer number of bytes used for initial alignment = "initial alignment"]
[ "initial alignment" number of bytes of value 0 ]
[ fingerprint 1, which  is "number of bytes per fingerprint" long ]
[ fingerprint 2 ]
[ fingerprint 3 ]
  ...
[ fingerprint N ]

byte_offset = i / 8;
bit_offset = i % 8;
bit = (fingerprint[byte_offset] >> bit_offset) & 1;

[ integer offset in DENS to first fingerprint with popcount=0 ]
[ integer offset  "  "   to first fingerprint with popcount=1 ]
   ...   
[ integer offset  "  "   to first fingerprint with popcount=N ]
[ integer offset in DENS to the byte after the last fingerprint with popcount=N ]

[NUL byte]
[ name 1 ]
[NUL byte]
[ name 2 ]
 ....
[ name N ]
[NUL byte]

[integer offset to name for fingerprint 1]
[integer offset to name for fingerprint 2]
[integer offset to name for fingerprint 3]
  ...
[integer offset to name for the last fingerprint]

Evil"; name="fred

<html>
<head><title>content-disposition test</title></head>
<body>
 <form method="POST" action="http://localhost:8888/" enctype="multipart/form-data">
  <input type="file" name="blah">
  <input type="submit">
 </form>
</body>
</html>

nc -l 8888

------WebKitFormBoundaryO5rrXim+NdIE0npI
Content-Disposition: form-data; name="blah"; filename="Evil"; name="fred"
Content-Type: application/octet-stream

-----------------------------94839879511149195311657737442
Content-Disposition: form-data; name="blah"; filename="Evil"; name="fred"
Content-Type: application/octet-stream

-----------------------------00000000000000000000000000000
Content-Disposition: form-data; name="blah"; filename="Evil"; name="fred"
Content-Type: text/plain; charset=us-ascii

------------lHwjN2s9agu9VAHOJ1ChbS
Content-Disposition: form-data; name="blah"; filename="default"
Content-Type: application/octet-stream

  public static List<Integer> generate(int n) {
    List<Integer> primes = new ArrayList<Integer>();

    for (int candidate = 2; n > 1; candidate++)
      for (; n%candidate == 0; n/=candidate)
        primes.add(candidate);

    return primes;

def fib(n):
    if n <= 1:
        return n
    else:
        return fib(n - 1) + fib(n - 2)

__main__

Andrew Dalke's writings

KNIME and beginners

Upcoming training classes (pre-announcement)

Sample test case for KNIME

What molecular properties can I compute? And how do I do it?

How accessible is KNIME to first-time users?

Can't compute the number of heavy atoms?

Instrumenting the AST

The AST module

Programmatically building an AST

The mystery of the wrong TypeError

Rewriting the AST for fun (and profit?)

(Well-known) Limitations in coverage.py

A different approach

Instrumenting the AST for code coverage

Try it out!

Pluses and minuses

"See also" and ruminations

Thanks!

Any comments?

New Cheminformatics Projects

Chemistry Toolkit Rosetta

Chem Fingerprints

Project hosting options?

The options

Must have mailing list and web page or wiki hosting

I want to try something other than SourceForge or a clone

Must support non-member access to a mailing list

Remaining options: KnowledgeForget and Project Kenai

Project Kenai

Show stopper - non-member access to the mailing list

No response to a feature requst

Just choose SourceForge?

Cheminformatics, bioinformatics, and system biology positions available

University College Cork, Ireland

Technical University of Denmark, on the north side of Copenhagen

Fingerprint File Format

Use cases

Design goals

FPF is based on the PNG block structure

FPF block types

FHDR: header block

tEXt (and potentially iTXt, zTXt): text block(s)

FDNS: dense fingerprint block

POPC: ordered population count offsets block

NAME block - list of fingerprint identifiers

NOFF block - offsets into the NAME block

Unresolved issues

Sorted or unsorted names?

Scaling

Storing multiple diverse fingerprint sets in the same file

Preserving order

Usefulness of Baldi

Fingerprint density

Streaming output

Development and future

Content-Disposition bug in browsers?

Reproducible

Safari 4.0.4

Firefox 3.5.5

Links 2.2

Opera 10.10

Newlines

Security Vulnerabilities

Problems with TDD

Test first vs. test last vs. good testing

Worked out TDD examples

Prime Factors

Fibonacci

Problem: TDD doesn't emphasize good test cases

Fibonacci Sequence

Problem: When do you add tests that should pass?

Problem: TDD does not consider worst-case scenarios

Prime Factors Kata

Fibonacci

Discussion

Problem: TDD doesn't give you confidence that the code works

Incorrect claim: TDD leads to 100% coverage

Complaint: TDD freezes the API too early

Conclusion