PySpark Demo

# https://dzone.com/articles/pyspark-dataframe-tutorial-introduction-to-datafra

dataDictionary = [

        ('James',{'hair':'black','eye':'brown'}),

        ('Michael',{'hair':'brown','eye':None}),

        ('Robert',{'hair':'red','eye':'black'}),

        ('Washington',{'hair':'red','eye':'grey'}),

        ('Jefferson',{'hair':'red','eye':''})

        ]

df = spark.createDataFrame(data=dataDictionary, schema = ["name","properties"])

df.printSchema()

df.columns

df.count()

df.select('name')

df.show(truncate=False)

dict2 = [

('James','James_last'),

('Michael','Michael_last'),

('Wendy', 'Wendy_last')

]

df2 = spark.createDataFrame(data=dict2, schema=['name','name_last'])

df2.show()

df.createOrReplaceTempView("PER")

spark.sql('select name from per where name like "J%"').show()

# https://www.educba.com/join-in-spark-sql/

df2.createOrReplaceTempView("PER_LAST")

spark.sql('select * from per p1 full outer join per_last p2 on p1.name=p2.name').show()

spark.sql('select p1.name from per p1 full outer join per_last p2 on p1.name=p2.name').rdd.map(lambda x:x['name']).collect()

# ['James', 'Washington', 'Michael', 'Robert', 'Jefferson', None]

#Create PySpark DataFrame from Pandas

sparkDF = spark.createDataFrame(pandasDF) 

Download SSL / TLS certificates using Python

import OpenSSL
import ssl

hostname='www.google.com'
port=443

cert = ssl.get_server_certificate((hostname, port))
x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)

x509.digest('sha256')
# b'0A:E6:46:01:80:9F:C7:33:84:19:6A:DD:6C:8E:5F:95:5C:F6:F2:75:46:32:1E:C9:61:1D:88:DA:9A:A9:B4:

x509.get_subject()
# <X509Name object '/C=US/ST=California/L=Mountain View/O=Google LLC/CN=www.google.com'

x509.get_notBefore().decode()                                                                                                                          # '20210311144716Z'

Anaconda / Miniconda - Python package managers

http://docs.continuum.io/anaconda/index.html
http://conda.pydata.org/miniconda.html

Anaconda is a free collection of powerful packages for Python that enables large-scale data management, analysis, and visualization for Business Intelligence, Scientific Analysis, Engineering, Machine Learning, and more.

Software carpentry

http://software-carpentry.org/v4/index.html

Who We Are

Our volunteers teach basic software skills to researchers in science, engineering, and medicine. Founded in 1998, we are now part of the Mozilla Science Lab.

What We Do

We run bootcamps all over the world, and provide open access material for self-paced instruction. We also run a training program for people who'd like to help us teach.

How To Help

Like all volunteer organizations, we depend on you to help us help others. You can host a bootcamp, help create new teaching materials, or improve the tools we use.

Python 2.5

try:
   ...
catch Exception, e:  # 2.6+ uses catch Exception as e
  ...


try:
    import json # only in Python 2.6+
except ImportError:
    import simplejson as json # but appengine only has Python 2.5

Puzzle finding shortest path

The first line has the number of rows in the nucleus (N), then the number of columns (M), then the length of Danny Dendrite's genome (X). Following that are N lines, each of which has M characters: an ‘o’ represents an empty square and an ‘x’ represents a filled square.

Print for him a path, consisting of a list of positions, starting anywhere in the nucleus, where:

- Each position is empty
- Each successive position is only 1 nm away from the one before it
- The path passes through each position in the nucleus at most once
- The number of positions visited is equal to the length of Danny Dendrite's genome (the starting and ending positions both count).

#
# P.T.
#
# https://dnanexus.com/careers/puzzles
#
#

import networkx # needs ver 1.6
import matplotlib.pyplot as plt

#input
rawData = """5 4 8
oxoo
xoxo
ooxo
xooo
oxox"""

def getNeighbours(row, col, maxRow, maxCol):
""">>> getNeighbours(1,1)
[[1, 0], [1, 2], [0, 1], [2, 1]]
"""
left = [row - 1, col]
right = [row + 1, col]
top = [row, col - 1]
bottom = [row, col + 1]

neighbours = [top, bottom, left, right]

if (top[1] < 0 or top[1] >= maxCol):
neighbours.remove(top)
if bottom[1] < 0 or bottom[1] >= maxCol:
neighbours.remove(bottom)
if left[0] < 0 or left[1] >= maxRow:
neighbours.remove(left)
if right[0] < 0 or right[0] >= maxRow:
neighbours.remove(right)

return neighbours

def isOk(node):
"""'o' isOk, 'x' is not"""
if (node == 'o'):
return True
else:
return False

# parse
#infile = [line.rstrip() for line in open('in').readlines()]
infile = [line.rstrip() for line in rawData.split('\n')]
[nrow, ncol, plen] = infile[0].split(' ')
nrow = int(nrow)
ncol = int(ncol)
plen = int(plen)
data = infile[1:]

# construct graph from input
G = networkx.Graph()
hash = '%s,%s'
print nrow, data, len(data)
assert len(data) == nrow
for row in range(nrow):
line = data[row]
assert len(line) == ncol
for col in range(ncol):
node = data[row][col]

#print 'node=', node, 'row=', row, 'col=', col

if (not isOk(node)):
continue

G.add_node(hash % (row,col))
neighbours = getNeighbours(row, col, nrow, ncol)

#print neighbours

for x,y in neighbours:

#print '?? x=', x, 'y=', y

if (not isOk(data[x][y])):
continue

#print 'x=', x, 'y=', y

G.add_edge(hash % (row,col), hash % (x,y))

print("Nodes\n%s" % G.nodes())
print("Edges\n%s" % G.edges())

# find shortest paths
paths = networkx.shortest_path(G)
for src in paths.keys():
for dest in paths[src].keys():
if len(paths[src][dest]) < plen:
continue
print("Path src=%s to dest=%s\n%s" % (src, dest, paths[src][dest]))

#print("Connected\n%s" % networkx.connected_components(G))

# visualize
networkx.draw(G)
plt.savefig("path.png")

NetworkX

http://networkx.lanl.gov/


>>> import networkx as nx

>>> G=nx.Graph()
>>> G.add_node("spam")
>>> G.add_edge(1,2)
>>> print(G.nodes())
[1, 2, 'spam']
>>> print(G.edges())
[(1, 2)]

SVD in Python

$ sudo apt-get install python-scipy python-matplotlib

from scipy import *
from pylab import *

# load img
img = imread('myimg.png')[:,:,0]
gray()
figure(1)
imshow(img)

# get A = U * S * Vt singular value decomposition
m,n = img.shape
U,S,Vt = svd(img)
S = resize(S,[m,1])*eye(m,n)

# get first 20 eigenvectors
k = 20
figure(2)
imshow(dot(U[:,1:k], dot(S[1:k,1:k], Vt[1:k,:])))
show()

http://www.cs.ubc.ca/~nando/540b-2011/lectures/l2.pdf

Python NetworkX

NetworkX is a Python package for the creation, manipulation, and study of the structure, dynamics, and functions of complex networks.

http://networkx.lanl.gov/

biopython and jython

$ jython setup.py build
$ jar cvf BioPython.jar -C build/lib ./

Include the produced jar file in your CLASSPATH or server build.

Have you run "jython setup.py install" yet? That will compile the *.py
files into Java class files, and should put them where jython looks.

See also:
http://www.jython.org/docs/using/cmdline.html#environment-variables

... from the biopython-dev mailist

PyCogent

Evolution

http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2375001/

Python convert String to List

>>> l=eval('[1,[2,4],3]')
>>> l
[1, [2, 4], 3]
>>> l[0]
1
>>> l[1]
[2, 4]

Python WSGI web test

Web Test
http://pythonpaste.org/webtest/

Werkzeug
http://werkzeug.pocoo.org/documentation/0.6/test.html

Python TimeComplexity

http://wiki.python.org/moin/TimeComplexity

Sorting dictionary values in Python

Python dictionary value sorting:
http://wiki.python.org/moin/HowTo/Sorting#Sortingbykeys

If the invocation of key returns a tuple, second and subsequent items in the tuple will be treated as sub-keys in the same way that Python generally sorts tuples:

>>> L = [('d', 2), ('a', 4), ('b', 3), ('c', 2)]
>>> sorted(L, key=lambda x:(x[1], x[0]))
[('c', 2), ('d', 2), ('b', 3), ('a', 4)]

igraph - python graph library

http://igraph.sourceforge.net/index.html

Introduction

igraph is a free software package for creating and manipulating undirected and directed graphs. It includes implementations for classic graph theory problems like minimum spanning trees and network flow, and also implements algorithms for some recent network analysis methods, like community structure search.

Features

igraph contains functions for generating regular and random graphs, manipulating graphs, assigning attributes to vertices and edges. It can calculate various structural properties, graph isomorphism, includes heuristics for community structure detection, supports many file formats. The R and Python interfaces support visualization.

Python logging

Python Logging
http://blog.tplus1.com/index.php/2007/09/28/the-python-logging-module-is-much-better-than-print-statements/

import logging

# Log everything, and send it to stderr.
logging.basicConfig(level=logging.DEBUG)

def g():
1/0

def f():
logging.debug("Inside f!")
try:
g()
except Exception, ex:
logging.exception("Something awful happened!")
logging.debug("Finishing f!")

if __name__ == "__main__":
f()

Python built-in functions

http://docs.python.org/library/functions.html

I heard they're pretty fast

map - maybe better for parallelization purposes
---
>>> l=[1,2,3,4,5]
>>> def dbl(x):
return x*2

>>> map(dbl, l)
[2, 4, 6, 8, 10]

or just
[x*2 for x in l]

set
---
>>> set([1,1,2,2,3,3])
set([1,2,3])

http://stackoverflow.com/questions/672172/how-to-use-python-map-and-other-functional-tools

PuLP - Linear Programming modules for Python

http://www.purplemath.com/modules/linprog.htm
http://code.google.com/p/pulp-or/wiki/OptimisationWithPuLP

http://mail.python.org/pipermail/python-announce-list/2005-May/003989.html
# Borrowed code from Jean-Sebastien, see link above ...
Example script:
--------------------------------------------------
from pulp import *

prob = LpProblem("test1", LpMinimize)

# Variables
x = LpVariable("x", 0, 4)
y = LpVariable("y", -1, 1)
z = LpVariable("z", 0)

# Objective
prob += x + 4*y + 9*z

# Constraints
prob += x+y <= 5
prob += x+z >= 10
prob += -y+z == 7

GLPK().solve(prob)

# Solution
for v in prob.variables():
print v.name, "=", v.varValue

print "objective=", value(prob.objective)
--------------------------------------------------
http://www.gnu.org/software/glpk/glpk.html

Linear programming is good for maximization or minimization problems or solving Sudoku puzzles! (http://130.216.209.237/engsci392/pulp/SudokuAsAnLP)

Works by incorporating linear equations ie y = mx + b (objective function) and a bunch of constraints eg 3*x - y <= 3 or something
works like magic!

As for applications, one example is with Punyakanok et al. '04 where he applies it to optimizing the best set of argument labels in semantic role labeling (SRL)
http://l2r.cs.uiuc.edu/~danr/Papers/PRYZ04.pdf

python best practices

http://www.fantascienza.net/leonardo/ar/python_best_practices.html

# Python profiler:
python -m profile -o stats myscript.py
>>> import pstats
>>> p = pstats.Stats('stats')
>>> p.sort_stats('time').print_stats(15)

# For source code with not 7-bit ASCII
add this on top:
# -*- coding: UTF-8 -*-
# Or just, if you have less memory:
# coding: latin

# Use iter* methods when possible
mapping = {5: "5", 6: "6"}
for key, val in mapping.iteritems(): ...
for key in mapping: ...

a = 5
b = 6
a, b = b, a # swap

a = b = c = 5

if x == 1: y = fun1(x)
elif x == 2: y = fun2(x)
elif x == 3: y = fun3(x)
else: y = None
# But sometimes a dict is better:
funs = {1: fun1, 2: fun2, 3: fun3}
y = funs.get(x, lambda x:None)(x)

def mul(x, y):
return x * y
l = [2, 3]
print mul(*l)

# Generally getters and setters are not used.
# Instance names starting with _ are meant as
# 'to not mess with' by convention.
# Instance names starting with __ are private
# and receive name mangling.
class Foo(object):
def __init__(self, x, y, z):
self.x_public = x
self._y_private = y
self.__z_veryprivate = z
print Foo(1, 2, 3).x_public

finder = re.compile(r"""
^ \s* # start at beginning+ opt spaces
( [\[\]] ) # Group 1: opening bracket
\s* # optional spaces
( [-+]? \d+ ) # Group 2: first number
\s* , \s* # opt spaces+ comma+ opt spaces
( [-+]? \d+ ) # Group 3: second number
\s* # opt spaces
( [\[\]] ) # Group 4: closing bracket
\s* $ # opt spaces+ end at the end
""", flags=re.VERBOSE)
# Sometimes it's positive to indent logically those
# lines just like code.

# Sometimes it can be positive to compose REs:
spaces = r"\s*" # optional spaces
number = r"( [-+]? \d+ )" # Group
bracket = r"( [\[\]] )" # Group. Closing bracket
parts = ["^", bracket, number, ",", number, bracket, "$"]
finder = re.compile(spaces.join(parts), flags=re.VERBOSE)


# Use doctests (or module tests):
def function(data):
"""A comment

>>> function()
None
>>> function(1)
result1
>>> function("a")
Traceback (most recent call last):
...
TypeError
"""
...implementation...

if __name__ == "__main__":
import doctest
doctest.testmod()
print "Tests done."


x = (1, 2, 6, 55, 63, 96, 125, 256,
301, 456, 958, 1256,
1359, 2568, 3597)
# Too much long lines must be broken with \
# but \ isn't necessary inside () [] {}



# Using psyco array.array of double and
# signed long become very fast
import array
a = array.array("d", [3.56, 2.12])