# Notebook from Siouffy/jupyter-ds
Path: data/PixieDustAndEthNotebook.ipynb

spark_____no_output_____import datetime

# spark imports
from pyspark.sql.types import StructType, TimestampType, DoubleType, LongType
from pyspark.sql.functions import udf

# importing pixiedust and df_profiling
import pixiedust
import spark_df_profiling

pixiedust.enableJobMonitor()  # monitor job progress

def eth_schema():
    return StructType() \
        .add('epoch', LongType()) \
        .add('usd', DoubleType())

parse_timestamp_udf = udf(lambda ts: datetime.datetime.fromtimestamp(ts), TimestampType())Pixiedust database opened successfully
Table VERSION_TRACKER created successfully

## Example Using Dataframe Profiling Tool _____no_output_____

ether_df = spark.read \
    .csv('/usr/notebooks/data/etherprice.csv', sep=';', header='false', schema=eth_schema()) \
    .withColumn('date', parse_timestamp_udf('epoch')) \
    .cache()  # caching dataframe for more efficient df_profiling (multiple passes)

# display the profiling report directly in the notebook
spark_df_profiling.ProfileReport(ether_df)

# to generate a HTML report file, save the ProfileReport to an object and use the .to_file() method:
# profile = spark_df_profiling.ProfileReport(df_spark)
# profile.to_file(outputfile="/tmp/myoutputfile.html"_____no_output_____

## Example using PixieDust Display utitlity _____no_output_____

display(ether_df)_____no_output_____%pixiedustLog -l debug2017-07-30 14:24:37,665 - pixiedust.utils.storage - INFO - Change in version detected: None -> 1.0.5.
2017-07-30 14:25:06,405 - pixiedust.display.chart - WARNING - Unable to import renderer seaborn: No module named 'seaborn'
2017-07-30 14:25:15,896 - pixiedust.utils.scalaBridge.PixiedustScalaMagics - DEBUG - Calling scala compiler with command: /root/scala/scala-2.11.8/bin/scalac -classpath /root/pixiedust/data/libs/*:/root/spark/spark-2.1.0-bin-hadoop2.7/conf/:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-dbcp-1.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-repl_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/lz4-1.3.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jersey-common-2.22.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/metrics-jvm-3.1.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jta-1.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/opencsv-2.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/xbean-asm5-shaded-4.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/antlr4-runtime-4.5.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-common-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/curator-recipes-2.6.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/JavaEWAH-0.3.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/oro-2.0.8.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/stringtemplate-3.2.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/xz-1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jets3t-0.9.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-mesos_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-hive-thriftserver_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jaxb-api-2.2.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/htrace-core-3.1.0-incubating.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/minlog-1.3.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/scala-compiler-2.11.8.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/apacheds-i18n-2.0.0-M15.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/datanucleus-core-3.2.10.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jersey-server-2.22.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-hive_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-tags_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-mapreduce-client-core-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/javax.inject-1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/netty-all-4.0.42.Final.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-annotations-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-mapreduce-client-app-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jackson-databind-2.6.5.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/javassist-3.18.1-GA.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/eigenbase-properties-1.1.5.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jetty-6.1.26.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/parquet-encoding-1.8.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/aopalliance-1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jetty-util-6.1.26.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jpam-1.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/parquet-common-1.8.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/avro-mapred-1.7.7-hadoop2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-compiler-3.0.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/snappy-java-1.1.2.6.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spire-macros_2.11-0.7.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-streaming_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/api-asn1-api-1.0.0-M20.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-network-shuffle_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jackson-xc-1.9.13.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/calcite-core-1.2.0-incubating.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/scalap-2.11.8.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jackson-mapper-asl-1.9.13.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/json4s-jackson_2.11-3.2.11.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jackson-annotations-2.6.5.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/netty-3.8.0.Final.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-digester-1.8.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/javax.servlet-api-3.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/parquet-hadoop-bundle-1.6.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/metrics-graphite-3.1.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/javax.ws.rs-api-2.0.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-cli-1.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-sql_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-yarn-client-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/kryo-shaded-3.0.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/java-xmlbuilder-1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/base64-2.3.8.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jackson-jaxrs-1.9.13.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-net-2.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-yarn-server-common-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-network-common_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hive-jdbc-1.2.1.spark2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/mail-1.4.7.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/osgi-resource-locator-1.0.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/RoaringBitmap-0.5.11.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jsp-api-2.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jul-to-slf4j-1.7.16.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-sketch_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/antlr-runtime-3.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-mapreduce-client-jobclient-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/httpcore-4.4.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jackson-module-scala_2.11-2.6.5.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/mesos-1.0.0-shaded-protobuf.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/datanucleus-api-jdo-3.2.6.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/activation-1.1.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/calcite-avatica-1.2.0-incubating.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-catalyst_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/xercesImpl-2.9.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/curator-framework-2.6.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-beanutils-1.7.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-yarn_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/gson-2.2.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hive-exec-1.2.1.spark2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/javax.annotation-api-1.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/parquet-column-1.8.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/bcprov-jdk15on-1.51.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/zookeeper-3.4.6.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/pmml-model-1.2.15.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/mx4j-3.0.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/json4s-core_2.11-3.2.11.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/bonecp-0.8.0.RELEASE.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/snappy-0.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jtransforms-2.4.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/httpclient-4.5.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/paranamer-2.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/xmlenc-0.52.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/parquet-format-2.3.0-incubating.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/pyrolite-4.13.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-configuration-1.6.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/log4j-1.2.17.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/chill_2.11-0.8.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-codec-1.10.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hk2-utils-2.4.0-b34.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/super-csv-2.2.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-mllib_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-yarn-api-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/curator-client-2.6.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/guice-3.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-compress-1.4.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-crypto-1.0.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jackson-core-2.6.5.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/protobuf-java-2.5.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jsr305-1.3.9.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/parquet-hadoop-1.8.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/slf4j-log4j12-1.7.16.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/ST4-4.0.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jackson-module-paranamer-2.6.5.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-client-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/metrics-core-3.1.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-mapreduce-client-common-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-unsafe_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-lang-2.6.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hive-cli-1.2.1.spark2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/scala-parser-combinators_2.11-1.0.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-mllib-local_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jersey-container-servlet-2.22.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/compress-lzf-1.0.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/validation-api-1.1.0.Final.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-math3-3.4.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-auth-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/chill-java-0.8.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jersey-container-servlet-core-2.22.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/stream-2.7.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jodd-core-3.5.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/libfb303-0.9.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-mapreduce-client-shuffle-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/scala-xml_2.11-1.0.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-lang3-3.5.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/slf4j-api-1.7.16.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/javax.inject-2.4.0-b34.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/stax-api-1.0-2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/calcite-linq4j-1.2.0-incubating.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/core-1.1.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-launcher_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-beanutils-core-1.8.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/api-util-1.0.0-M20.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/leveldbjni-all-1.8.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jersey-media-jaxb-2.22.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-yarn-server-web-proxy-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-yarn-common-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/stax-api-1.0.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/metrics-json-3.1.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/javolution-5.5.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/breeze_2.11-0.12.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/guice-servlet-3.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/objenesis-2.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/avro-ipc-1.7.7.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/antlr-2.7.7.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/ivy-2.4.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jcl-over-slf4j-1.7.16.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hive-beeline-1.2.1.spark2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jline-2.12.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/arpack_combined_all-0.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/apacheds-kerberos-codec-2.0.0-M15.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/guava-14.0.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/univocity-parsers-2.2.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jdo-api-3.0.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-core_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-pool-1.5.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/libthrift-0.9.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hk2-api-2.4.0-b34.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/avro-1.7.7.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/scala-library-2.11.8.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/shapeless_2.11-2.0.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/janino-3.0.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-io-2.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/breeze-macros_2.11-0.12.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jackson-core-asl-1.9.13.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jersey-client-2.22.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hadoop-hdfs-2.7.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spark-graphx_2.11-2.1.0.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hive-metastore-1.2.1.spark2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/aopalliance-repackaged-2.4.0-b34.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/jersey-guava-2.22.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/joda-time-2.9.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-httpclient-3.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/hk2-locator-2.4.0-b34.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-collections-3.2.2.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/derby-10.12.1.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/spire_2.11-0.7.4.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/json4s-ast_2.11-3.2.11.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/apache-log4j-extras-1.2.17.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/scala-reflect-2.11.8.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/pmml-schema-1.2.15.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/datanucleus-rdbms-3.2.9.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/parquet-jackson-1.8.1.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/commons-logging-1.1.3.jar:/root/spark/spark-2.1.0-bin-hadoop2.7/jars/py4j-0.10.4.jar pixiedustRunner.scala
2017-07-30 14:26:09,076 - pixiedust.utils.sparkJobProgressMonitor - ERROR - Unexpected error while adding Spark Listener
Traceback (most recent call last):
  File "/opt/conda/lib/python3.5/site-packages/pixiedust/utils/sparkJobProgressMonitor.py", line 203, in addSparkListener
    _env.getTemplate("sparkJobProgressMonitor/addSparkListener.scala").render()
  File "/opt/conda/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2115, in run_cell_magic
    result = fn(magic_arg_s, cell)
  File "<decorator-gen-123>", line 2, in scala
  File "/opt/conda/lib/python3.5/site-packages/IPython/core/magic.py", line 188, in <lambda>
    call = lambda f, *a, **k: f(*a, **k)
  File "/opt/conda/lib/python3.5/site-packages/pixiedust/utils/scalaBridge.py", line 178, in scala
    self.getLineOption(line, "channel"), self.getLineOption(line, "receiver"))
  File "/opt/conda/lib/python3.5/site-packages/pixiedust/utils/javaBridge.py", line 51, in __init__
    self.captureOutput(captureOutput)
  File "/opt/conda/lib/python3.5/site-packages/pixiedust/utils/javaBridge.py", line 71, in captureOutput
    pixiedustOutputSink = JavaWrapper("com.ibm.pixiedust.PixiedustOutputStream").jHandle(self.outputChannel)
TypeError: 'JavaPackage' object is not callable
2017-07-30 14:30:40,910 - pixiedust.display - DEBUG - Creating a new display handler with options {}: <pixiedust.display.display.CellHandshakeMeta object at 0x7f739b0e4550>
2017-07-30 14:30:41,551 - pixiedust.display.display.Display - DEBUG - pixiedust code: if (typeof pixiedust == "undefined"){
    var s=document.getElementsByTagName('script')[0];
    var g=document.createElement( 'script'  );
    g.type= 'text/javascript';
    g.defer=false;
    g.async=false;
    var code = "var pixiedust = (function(){\n"+
"    return {\n"+
"        \n"+
"        executeDisplay:function(pd_controls, user_controls){\n"+
"            pd_controls = pd_controls || {};\n"+
"            user_controls = user_controls || {\"options\":{}};\n"+
"            var options = $.extend({}, pd_controls.options || {}, user_controls.options || {} );\n"+
"            function onDisplayDone(){\n"+
"                if (user_controls.onDisplayDone){\n"+
"                    user_controls.onDisplayDone();\n"+
"                }\n"+
"            }\n"+
"            var pd_prefix = pd_controls.prefix;\n"+
"            var $targetDivId = user_controls.targetDivId;\n"+
"            !function() {\n"+
"    function getTargetNode(){\n"+
"        return $('#' + ($targetDivId || (\"wrapperHTML\"+ pd_prefix)));\n"+
"    }\n"+
"    var cellId = options.cell_id || \"\";\n"+
"    var curCell=IPython.notebook.get_cells().filter(function(cell){\n"+
"        return cell.cell_id==cellId;\n"+
"    });\n"+
"    curCell=curCell.length>0?curCell[0]:null;\n"+
"    console.log(\"curCell\",curCell);\n"+
"    var startWallToWall;\n"+
"        var callbacks = {\n"+
"        shell : {\n"+
"            reply : function(){\n"+
"                if ( !callbacks.response ){\n"+
"                    getTargetNode().html(\"\")\n"+
"                    if (user_controls.onDisplayDone){\n"+
"                        user_controls.onDisplayDone();\n"+
"                    }\n"+
"                }\n"+
"            },\n"+
"            payload : {\n"+
"                set_next_input : function(payload){\n"+
"                    if (curCell){\n"+
"                        curCell._handle_set_next_input(payload);\n"+
"                    }\n"+
"                }\n"+
"            }\n"+
"        },\n"+
"        iopub:{\n"+
"            output:function(msg){\n"+
"                callbacks.response = true;\n"+
"                console.log(\"msg\", msg);\n"+
"                if (cellId == \"\"){\n"+
"                    if (curCell){\n"+
"                        curCell.output_area.handle_output.apply(curCell.output_area, arguments);\n"+
"                        curCell.output_area.outputs=[];\n"+
"                    }else{\n"+
"                        console.log(\"Could not find current cell\");\n"+
"                    }\n"+
"                    return;\n"+
"                }\n"+
"                var msg_type=msg.header.msg_type;\n"+
"                var content = msg.content;\n"+
"                if(msg_type===\"stream\"){\n"+
"                    if (user_controls.onSuccess){\n"+
"                        user_controls.onSuccess(content.text);\n"+
"                    }else{\n"+
"                        getTargetNode().html(content.text);\n"+
"                    }\n"+
"                }else if (msg_type===\"display_data\" || msg_type===\"execute_result\"){\n"+
"                    var html=null;\n"+
"                    if (!!content.data[\"text/html\"]){\n"+
"                        html=content.data[\"text/html\"];\n"+
"                    }else if (!!content.data[\"image/png\"]){\n"+
"                        html=html||\"\";\n"+
"                        html+=\"\";\n"+
"                    }\n"+
"                                                    \n"+
"                    if (!!content.data[\"application/javascript\"]){\n"+
"                        try {\n"+
"                            eval(content.data[\"application/javascript\"]);\n"+
"                        } catch(err) {\n"+
"                            curCell.output_area.handle_output.apply(curCell.output_area, arguments);\n"+
"                        }                        \n"+
"                        return;\n"+
"                    }\n"+
"                    \n"+
"                    if (html){\n"+
"                        try{\n"+
"                            if (user_controls.onSuccess){\n"+
"                                user_controls.onSuccess(html);\n"+
"                            }else{\n"+
"                                getTargetNode().html(html);\n"+
"                            }\n"+
"                        }catch(e){\n"+
"                            console.log(\"Invalid html output\", e, html);\n"+
"                            getTargetNode().html( \"Invalid html output: \" + e.message + \"
\" \n"+
"                                + html.replace(/>/g,'>').replace(/</g,'<').replace(/\"/g,'"') + \"
\");\n"+
"                        }\n"+
"\n"+
"                        if(curCell&&curCell.output_area&&curCell.output_area.outputs){\n"+
"                            setTimeout(function(){\n"+
"                                var data = JSON.parse(JSON.stringify(content.data));\n"+
"                                if(!!data[\"text/html\"])data[\"text/html\"]=html;\n"+
"                                function savedData(data){\n"+
"                                    \n"+
"                                    var markup='';\n"+
"                                    markup+='
Hey, there\\'s something awesome here! To see it, open this notebook outside GitHub, in a viewer like Jupyter
';\n"+
"                                    nodes = $.parseHTML(data[\"text/html\"], null, true);\n"+
"                                    var s = $(nodes).wrap(\"
\").parent().find(\".pd_save\").not(\".pd_save .pd_save\")\n"+
"                                    s.each(function(){\n"+
"                                        var found = false;\n"+
"                                        if ( $(this).attr(\"id\") ){\n"+
"                                            var n = $(\"#\" + $(this).attr(\"id\"));\n"+
"                                            if (n.length>0){\n"+
"                                                found=true;\n"+
"                                                n.each(function(){\n"+
"                                                    $(this).addClass(\"is-viewer-good\");\n"+
"                                                });\n"+
"                                                markup+=n.wrap(\"
\").parent().html();\n"+
"                                            }\n"+
"                                        }else{\n"+
"                                            $(this).addClass(\"is-viewer-good\");\n"+
"                                        }\n"+
"                                        if (!found){\n"+
"                                            markup+=$(this).parent().html();\n"+
"                                        }\n"+
"                                    });\n"+
"                                    data[\"text/html\"] = markup;\n"+
"                                    return data;\n"+
"                                }\n"+
"                                curCell.output_area.outputs.push({\"data\": savedData(data),\"metadata\":content.metadata,\"output_type\":msg_type});\n"+
"                            },2000);\n"+
"                        }\n"+
"                    }\n"+
"                }else if (msg_type === \"error\") {\n"+
"                    require(['base/js/utils'], function(utils) {\n"+
"                        var tb = content.traceback;\n"+
"                        console.log(\"tb\",tb);\n"+
"                        if (tb && tb.length>0){\n"+
"                            var data = tb.reduce(function(res, frame){return res+frame+'\\\\n';},\"\");\n"+
"                            console.log(\"data\",data);\n"+
"                            data = utils.fixConsole(data);\n"+
"                            data = utils.fixCarriageReturn(data);\n"+
"                            data = utils.autoLinkUrls(data);\n"+
"                            if (user_controls.onError){\n"+
"                                user_controls.onError(data);\n"+
"                            }else{\n"+
"                                getTargetNode().html(\"
\" + data +\"
\");\n"+
"                            }\n"+
"                        }\n"+
"                    });\n"+
"                }else{\n"+
"                    callbacks.response = false;\n"+
"                }\n"+
"                if (user_controls.onDisplayDone){\n"+
"                    user_controls.onDisplayDone();\n"+
"                }\n"+
"            }\n"+
"        }\n"+
"    }\n"+
"    \n"+
"    if (IPython && IPython.notebook && IPython.notebook.session && IPython.notebook.session.kernel){\n"+
"        var command = user_controls.script || pd_controls.command.replace(\"cellId\",cellId);\n"+
"        if ( !user_controls.script){\n"+
"            function addOptions(options, override=true){\n"+
"                function getStringRep(v) {\n"+
"                    return \"'\" + v + \"'\";\n"+
"                }\n"+
"                for (var key in (options||{})){\n"+
"                    var value = options[key];\n"+
"                    var hasValue = value != null && typeof value !== 'undefined' && value !== '';\n"+
"                    var replaceValue = hasValue ? (key+\"=\" + getStringRep(value) ) : \"\";\n"+
"                    var pattern = (hasValue?\"\":\",\")+\"\\\\s*\" + key + \"\\\\s*=\\\\s*'(\\\\\\\\'|[^'])*'\";\n"+
"                    var rpattern=new RegExp(pattern);\n"+
"                    var n = command.search(rpattern);\n"+
"                    if ( n >= 0 ){\n"+
"                        if (override){\n"+
"                            command = command.replace(rpattern, replaceValue);\n"+
"                        }\n"+
"                    }else if (hasValue){\n"+
"                        var n = command.lastIndexOf(\")\");\n"+
"                        command = [command.slice(0, n), (command[n-1]==\"(\"? \"\":\",\") + replaceValue, command.slice(n)].join('')\n"+
"                    }        \n"+
"                }\n"+
"            }\n"+
"            if(typeof cellMetadata != \"undefined\" && cellMetadata.displayParams){\n"+
"                addOptions(cellMetadata.displayParams);\n"+
"                addOptions({\"showchrome\":\"true\"});\n"+
"            }else if (curCell && curCell._metadata.pixiedust ){\n"+
"                addOptions(curCell._metadata.pixiedust.displayParams || {}, pd_controls.useCellMetadata);\n"+
"            }\n"+
"            addOptions(user_controls.options||{});\n"+
"            var pattern = \"\\\\w*\\\\s*=\\\\s*'(\\\\\\\\'|[^'])*'\";\n"+
"            var rpattern=new RegExp(pattern,\"g\");\n"+
"            var n = command.match(rpattern);\n"+
"            \n"+
"            var org_params = {}\n"+
"            for (var i=0; i<n.length;i++){\n"+
"                var parts = n[i].split(\"=\")\n"+
"                if (parts[0].trim() == \"org_params\"){\n"+
"                    var value = parts[1].trim()\n"+
"                    var values = value.substring(1,value.length-1).split(\",\");\n"+
"                    for (var p in values){\n"+
"                        org_params[values[p].trim()] = true;\n"+
"                    }\n"+
"                    break;\n"+
"                }\n"+
"            }\n"+
"            var displayParams={}\n"+
"            for (var i = 0; i < n.length; i++){\n"+
"                var parts=n[i].split(\"=\");\n"+
"                var key = parts[0].trim();\n"+
"                var value = parts[1].trim()\n"+
"                if (!key.startsWith(\"nostore_\") && key != \"showchrome\" && key != \"prefix\" && key != \"cell_id\" && key != \"org_params\" && !!!org_params[key]){\n"+
"                    displayParams[key] = value.substring(1,value.length-1);\n"+
"                }\n"+
"            }\n"+
"\n"+
"            \n"+
"        }\n"+
"        if(curCell&&curCell.output_area){\n"+
"            if ( !user_controls.nostoreMedatadata ){\n"+
"                curCell._metadata.pixiedust = curCell._metadata.pixiedust || {}\n"+
"                curCell._metadata.pixiedust.displayParams=displayParams\n"+
"                curCell.output_area.outputs=[];\n"+
"                var old_msg_id = curCell.last_msg_id;\n"+
"                if (old_msg_id) {\n"+
"                    curCell.kernel.clear_callbacks_for_msg(old_msg_id);\n"+
"                }\n"+
"            }\n"+
"        }else{\n"+
"            console.log(\"couldn't find the cell\");\n"+
"        }\n"+
"        $('#wrapperJS' + pd_prefix).html(\"\")\n"+
"        if (!getTargetNode().hasClass( \"no_loading_msg\" )){\n"+
"            getTargetNode().html(\n"+
"                '
'+\n"+
"                    ''+\n"+
"                '
'+\n"+
"                '
' +\n"+
"                    (getTargetNode().attr(\"pd_loading_msg\") || \"Loading your data. Please wait...\") +\n"+
"                '
'\n"+
"            );\n"+
"        }\n"+
"        console.log(\"Running command2\",command);\n"+
"        IPython.notebook.session.kernel.execute(command, callbacks, {silent:true,store_history:false,stop_on_error:true});\n"+
"    }\n"+
"}()\n"+
"        },\n"+
"        executeInDialog:function(pd_controls, user_controls){\n"+
"            pd_controls = pd_controls || {};\n"+
"            user_controls = user_controls || {\"options\":{}};\n"+
"            var displayOptions = $.extend({}, pd_controls.options || {}, user_controls.options || {} );\n"+
"            var global={};\n"+
"            require(['base/js/dialog'],function(dialog){\n"+
"                var modal = dialog.modal;\n"+
"                var attr_pd_ctrl = JSON.stringify(pd_controls).trim()\n"+
"                    .replace(/&/g, '&')\n"+
"                    .replace(/'/g, ''')\n"+
"                    .replace(/\"/g, '"')\n"+
"                    .replace(/</g, '<')\n"+
"                    .replace(/>/g, '>');\n"+
"                var dialogRoot = \"dialog\" + pd_controls.prefix + \"root\";\n"+
"                var options = {\n"+
"                    title: \"Pixiedust: \" + (displayOptions.title || \"Dialog\"),\n"+
"                    body: '

',\n"+
"                    sanitize:false,\n"+
"                    notebook: IPython.notebook,\n"+
"                    keyboard_manager: IPython.notebook.keyboard_manager,\n"+
"                    maximize_modal: (displayOptions.maximize === \"true\"),\n"+
"                    buttons: {\n"+
"                        OK: {\n"+
"                            class : \"btn-primary btn-ok\",\n"+
"                            click: function() {\n"+
"                                var dlg = $(\"#\" + dialogRoot + \" > pd_dialog\");\n"+
"                                try{\n"+
"                                    return new Function('global', 'modal_obj', dlg.find(\"> pd_ok\").text().trim())(global, modal_obj);\n"+
"                                }catch(e){\n"+
"                                    console.error(e);\n"+
"                                    return false;\n"+
"                                }\n"+
"                            }\n"+
"                        },\n"+
"                        Cancel: {\n"+
"                            class : \"btn-cancel\",\n"+
"                            click: function(){\n"+
"                            }\n"+
"                        }\n"+
"                    }\n"+
"                };\n"+
"\n"+
"                function resizeDialog() {\n"+
"                    global.modalBodyStyle = $('.pixiedust .modal-body').attr('style');\n"+
"                    global.modalFooterStyle = $('.pixiedust .modal-footer').attr('style');\n"+
"                    $('.pixiedust .modal-body').attr('style', global.modalBodyStyle ? global.modalBodyStyle + ';padding:5px 20px !important;' : 'padding:5px 20px !important;');\n"+
"                    $('.pixiedust .modal-footer').attr('style', 'display:none !important;');\n"+
"                };\n"+
"\n"+
"                function resetDialog() {\n"+
"                    if (global.modalBodyStyle) {\n"+
"                        $('.pixiedust .modal-body').attr('style', global.modalBodyStyle);\n"+
"                    } else {\n"+
"                        $('.pixiedust .modal-body').removeAttr('style');\n"+
"                    }\n"+
"                    if (global.modalFooterStyle) {\n"+
"                        $('.pixiedust .modal-footer').attr('style', global.modalFooterStyle);\n"+
"                    } else {\n"+
"                        $('.pixiedust .modal-footer').removeAttr('style');\n"+
"                    }\n"+
"                };\n"+
"\n"+
"                var modal_obj = modal(options);\n"+
"                modal_obj.addClass('pixiedust pixiedust-app');\n"+
"                if (options.maximize_modal) {\n"+
"                    modal_obj.addClass('pixiedust pixiedust-app pixiedust-maximize');\n"+
"                }\n"+
"                modal_obj.on('shown.bs.modal', function(){\n"+
"                    resizeDialog();\n"+
"                    var isFF = navigator.userAgent.toLowerCase().indexOf('firefox') > -1;\n"+
"                    if( isFF && options.keyboard_manager){\n"+
"                        \n"+
"                        var KeyboardManager = require('notebook/js/keyboardmanager').KeyboardManager;\n"+
"                        global.KMEnableProto = KeyboardManager.prototype.enable;\n"+
"                        KeyboardManager.prototype.enable = function () {\n"+
"                            this.enabled = false;\n"+
"                        };\n"+
"                    }\n"+
"                    IPython.keyboard_manager.register_events(modal_obj);\n"+
"                    user_controls.options.targetDivId = user_controls.targetDivId = dialogRoot;\n"+
"                    if ( user_controls.options.dialog == 'true'){\n"+
"                        user_controls.onDisplayDone = function(){\n"+
"                            var dlg = $(\"#\" + dialogRoot + \" > pd_dialog\")\n"+
"                            try{\n"+
"                                new Function('global', 'modal_obj', dlg.find(\"> pd_onload\").text().trim())(global, modal_obj);\n"+
"                            }catch(e){\n"+
"                                console.error(e);\n"+
"                            }\n"+
"                        }\n"+
"                    }\n"+
"                    pixiedust.dialogRoot = dialogRoot;\n"+
"                    pixiedust.executeDisplay(pd_controls, user_controls);\n"+
"                });\n"+
"                modal_obj.on(\"hidden.bs.modal\", function () {\n"+
"                    resetDialog();\n"+
"                    if ( global.KMEnableProto ){\n"+
"                        var KeyboardManager = require('notebook/js/keyboardmanager').KeyboardManager;\n"+
"                        KeyboardManager.prototype.enable = global.KMEnableProto;\n"+
"                        delete global.KMEnableProto;\n"+
"                    }\n"+
"                    pixiedust.dialogRoot = null;\n"+
"                });\n"+
"            })\n"+
"        }\n"+
"    }\n"+
"})();\n"+
"\n"+
"function resolveScriptMacros(script){\n"+
"    script = script && script.replace(/\\$val\\(\\\"?(\\w*)\\\"?\\)/g, function(a,b){\n"+
"        var v = $(\"#\" + b ).val();\n"+
"        if (!v && window[b] && typeof window[b] === \"function\"){\n"+
"            v = window[b]();\n"+
"        }\n"+
"        if (!v && pixiedust[b] && typeof pixiedust[b] === \"function\"){\n"+
"            v = pixiedust[b]();\n"+
"        }\n"+
"        if (!v){\n"+
"            console.log(\"Warning: Unable to resolve value for element \", b);\n"+
"            return a;\n"+
"        }\n"+
"        return v.split('\"').join('"').split('\\n').join('\\\\n');\n"+
"    });\n"+
"    return script;\n"+
"}\n"+
"\n"+
"function getParentScript(element){\n"+
"    var scripts = [];\n"+
"    \n"+
"    $(element).parents(\"[pd_script]\").each(function(){\n"+
"        scripts.unshift(this.getAttribute(\"pd_script\"));\n"+
"    });\n"+
"\n"+
"    \n"+
"    var script = \"\";\n"+
"    $.each( scripts, function(index, value){\n"+
"        if (value){\n"+
"            script += \"\\n\" + value;\n"+
"        }\n"+
"    });\n"+
"    return script;\n"+
"}\n"+
"\n"+
"function preRun(element){\n"+
"    var preRunCode = null;\n"+
"    $(element).find(\"> pd_script\").each(function(){\n"+
"        var type = this.getAttribute(\"type\");\n"+
"        if (type==\"preRun\"){\n"+
"            preRunCode = $(this).text();\n"+
"        }\n"+
"    });\n"+
"    if (!preRunCode ){\n"+
"        return true;\n"+
"    }\n"+
"    return new Function(preRunCode.trim())();\n"+
"}\n"+
"\n"+
"function addOptions(command, options, override=true){\n"+
"    function getStringRep(v) {\n"+
"        return \"'\" + v + \"'\";\n"+
"    }\n"+
"    for (var key in (options||{})){\n"+
"        var value = options[key];\n"+
"        var hasValue = value != null && typeof value !== 'undefined' && value !== '';\n"+
"        var replaceValue = hasValue ? (key+\"=\" + getStringRep(value) ) : \"\";\n"+
"        var pattern = (hasValue?\"\":\",\")+\"\\\\s*\" + key + \"\\\\s*=\\\\s*'(\\\\\\\\'|[^'])*'\";\n"+
"        var rpattern=new RegExp(pattern);\n"+
"        var n = command.search(rpattern);\n"+
"        if ( n >= 0 ){\n"+
"            if (override){\n"+
"                command = command.replace(rpattern, replaceValue);\n"+
"            }\n"+
"        }else if (hasValue){\n"+
"            var n = command.lastIndexOf(\")\");\n"+
"            command = [command.slice(0, n), (command[n-1]==\"(\"? \"\":\",\") + replaceValue, command.slice(n)].join('')\n"+
"        }        \n"+
"    }\n"+
"    return command;\n"+
"}\n"+
"\n"+
"function readExecInfo(pd_controls, element){\n"+
"    var execInfo = {}\n"+
"    execInfo.options = {}\n"+
"    var hasOptions = false;\n"+
"    $.each( element.attributes, function(){\n"+
"        if (this.name.startsWith(\"option_\")){\n"+
"            hasOptions = true;\n"+
"            execInfo.options[this.name.replace(\"option_\", \"\")] = this.value || null;\n"+
"        }\n"+
"    });\n"+
"    var pd_options = resolveScriptMacros(element.getAttribute(\"pd_options\"));\n"+
"    if (pd_options){\n"+
"        var parts = pd_options.split(\";\");\n"+
"        $.each( parts, function(){\n"+
"            var index = this.indexOf(\"=\");\n"+
"            if ( index > 1){\n"+
"                hasOptions = true;\n"+
"                execInfo.options[this.substring(0, index)] = this.substring(index+1);\n"+
"            }\n"+
"        });\n"+
"    }\n"+
"    execInfo.options.nostore_figureOnly = true;\n"+
"    execInfo.options.targetDivId = execInfo.targetDivId = element.getAttribute(\"pd_target\");\n"+
"    if (execInfo.options.targetDivId){\n"+
"        execInfo.options.no_margin=true;\n"+
"    }\n"+
"\n"+
"    execInfo.options.widget = element.getAttribute(\"pd_widget\");\n"+
"\n"+
"        var parentStyles = [];\n"+
"    var hiddenBlockStyle = 'visibility: hidden !important; display: block !important;';\n"+
"    var tDiv = $(\"#\" + execInfo.targetDivId);\n"+
"    var tDivParents = tDiv.parents().addBack().filter(':hidden');\n"+
"    tDivParents.each(function() {\n"+
"        var currentStyle = $(this).attr('style');\n"+
"        parentStyles.push(currentStyle);\n"+
"        $(this).attr('style', currentStyle ? currentStyle + ';' + hiddenBlockStyle : hiddenBlockStyle);\n"+
"    });\n"+
"\n"+
"        w = tDiv.width()\n"+
"    if (w) {\n"+
"        execInfo.options.nostore_cw= w;\n"+
"    }\n"+
"    if ($(element).parents(\".modal-dialog\").length > 0 ) {\n"+
"        h = tDiv.height()\n"+
"        if (h) {\n"+
"            execInfo.options.nostore_ch = h-10;\n"+
"        }\n"+
"    }\n"+
"\n"+
"        tDivParents.each(function(i) {\n"+
"        if (parentStyles[i] === undefined) {\n"+
"            $(this).removeAttr('style');\n"+
"        } else {\n"+
"            $(this).attr('style', parentStyles[i]);\n"+
"        }\n"+
"    });\n"+
"\n"+
"    execInfo.script = element.getAttribute(\"pd_script\");\n"+
"    if (!execInfo.script){\n"+
"        $(element).find(\"> pd_script\").each(function(){\n"+
"            var type = this.getAttribute(\"type\");\n"+
"            if (!type || type==\"python\"){\n"+
"                execInfo.script = $(this).text();\n"+
"            }\n"+
"        })\n"+
"    }\n"+
"\n"+
"    execInfo.refresh = element.hasAttribute(\"pd_refresh\");\n"+
"    execInfo.norefresh = element.hasAttribute(\"pd_norefresh\");\n"+
"    execInfo.entity = element.hasAttribute(\"pd_entity\") ? element.getAttribute(\"pd_entity\") || \"pixieapp_entity\" : null;\n"+
"\n"+
"    function applyEntity(c, e, doptions){\n"+
"        if (!e){\n"+
"            return addOptions(c, doptions);\n"+
"        }\n"+
"        c = c.replace(/\\((\\w*),/, \"($1.\" + e + \",\")\n"+
"        return addOptions(c, doptions);\n"+
"    }\n"+
"\n"+
"    if (!hasOptions && (execInfo.refresh || execInfo.options.widget) && !execInfo.script){\n"+
"        execInfo.script = \"#refresh\";\n"+
"    }\n"+
"\n"+
"    var dialog = (execInfo.options.dialog == \"true\");\n"+
"    if ( dialog ){\n"+
"        execInfo.script = execInfo.script || \"#refresh\";\n"+
"        execInfo.refresh = true;\n"+
"    }\n"+
"\n"+
"    if (execInfo.script){\n"+
"        execInfo.script = execInfo.script.trim()\n"+
"        \n"+
"        var match = pd_controls.command.match(/display\\((\\w*),/)\n"+
"        if (match){\n"+
"            var entity = match[1]\n"+
"            console.log(\"Inject self with entity\", entity)\n"+
"            execInfo.script = \"from pixiedust.utils.shellAccess import ShellAccess\\n\"+\n"+
"                \"self=ShellAccess['\" + entity + \"']\\n\" +\n"+
"                resolveScriptMacros( getParentScript(element) ) + '\\n' +\n"+
"                resolveScriptMacros(execInfo.script);\n"+
"            \n"+
"            if ( ( (!dialog && !execInfo.targetDivId) || execInfo.refresh || execInfo.entity) && !execInfo.norefresh && $(element).children(\"target[pd_target]\").length == 0){\n"+
"                \n"+
"                execInfo.script += \"\\n\" + applyEntity(pd_controls.command, execInfo.entity, execInfo.options)\n"+
"            }else{\n"+
"                \n"+
"                execInfo.targetDivId=execInfo.targetDivId || \"dummy\";\n"+
"            }\n"+
"        }else{\n"+
"            console.log(\"Unable to extract entity variable from command\", pd_controls.command);\n"+
"        }\n"+
"    }\n"+
"\n"+
"    if (!hasOptions && !execInfo.targetDivId && !execInfo.script){\n"+
"        return element.hasAttribute(\"pixiedust\")?null:readExecInfo(pd_controls, element.parentElement);\n"+
"    }\n"+
"\n"+
"    if (!execInfo.script){\n"+
"        execInfo.script = applyEntity(pd_controls.command, execInfo.entity, execInfo.options);\n"+
"    }\n"+
"\n"+
"    \n"+
"    execInfo.nostoreMedatadata = true;\n"+
"\n"+
"    \n"+
"    if ( pixiedust.dialogRoot ){\n"+
"        execInfo.targetDivId = execInfo.targetDivId || pixiedust.dialogRoot;\n"+
"    }\n"+
"\n"+
"    execInfo.execute = function(){\n"+
"        \n"+
"        if (!preRun(element)){\n"+
"            return;\n"+
"        }\n"+
"        if ( this.options.dialog == 'true' ){\n"+
"            pixiedust.executeInDialog(pd_controls, this);\n"+
"        }else{\n"+
"            pixiedust.executeDisplay(pd_controls, this);\n"+
"        }\n"+
"    }\n"+
"\n"+
"    console.log(\"execution info: \", execInfo);\n"+
"    return execInfo;\n"+
"}\n"+
"\n"+
"function runElement(element){\n"+
"    var pd_controls = element.getAttribute(\"pixiedust\");\n"+
"    if (!pd_controls){\n"+
"        $(element).parents(\"[pixiedust]\").each(function(){\n"+
"            pd_controls = pd_controls || this.getAttribute(\"pixiedust\");\n"+
"        });\n"+
"    }\n"+
"    var execQueue = [];\n"+
"    if (pd_controls){\n"+
"        pd_controls = JSON.parse(pd_controls);\n"+
"        \n"+
"        execQueue.push( readExecInfo(pd_controls, element) );\n"+
"\n"+
"        \n"+
"        $(element).children(\"target[pd_target]\").each(function(){\n"+
"            execQueue.push( readExecInfo(pd_controls, this))\n"+
"        });\n"+
"    }\n"+
"    return execQueue;\n"+
"}\n"+
"\n"+
"function filterNonTargetElements(element){\n"+
"    if (element && element.tagName == \"I\"){\n"+
"        return filterNonTargetElements(element.parentElement);\n"+
"    }\n"+
"    return element;\n"+
"}\n"+
"\n"+
"\n"+
"$(document).on( \"click\", \"[pixiedust]\", function(event){\n"+
"    execQueue = runElement(filterNonTargetElements(event.target));\n"+
"    \n"+
"    $.each( execQueue, function(index, value){\n"+
"        if (value){\n"+
"            event.stopImmediatePropagation();\n"+
"            value.execute();\n"+
"        }\n"+
"    });\n"+
"});\n"+
"\n"+
"$(document).on( \"DOMNodeInserted\", \"[pd_widget]\", function(event){\n"+
"    event.stopImmediatePropagation();\n"+
"    execQueue = runElement(filterNonTargetElements(event.target));\n"+
"    \n"+
"    $.each( execQueue, function(index, value){\n"+
"        if (value){\n"+
"            value.targetDivId = $(event.target).uniqueId().attr('id');\n"+
"            $(event.target).removeAttr(\"pd_widget\");\n"+
"            value.execute();\n"+
"        }\n"+
"    });\n"+
"});\n"+
"\n"+
"\n"+
"$(document).on(\"pd_event\", function(event, eventInfo){\n"+
"    targetDivId = eventInfo.targetDivId;\n"+
"    if (targetDivId){\n"+
"        eventHandlers = $(\"pd_event_handler\").filter(function(){\n"+
"            if (this.getAttribute(\"pd_target\") == targetDivId){\n"+
"                return true;\n"+
"            }\n"+
"            \n"+
"            return $(this).parents(\"[pd_target]\").filter(function(){\n"+
"                return this.getAttribute(\"pd_target\") == targetDivId;\n"+
"            }).length > 0;\n"+
"        });\n"+
"        eventHandlers.each(function(){\n"+
"            execQueue = runElement(this);\n"+
"            $.each( execQueue, function(index, value){\n"+
"                if (value){\n"+
"                    \n"+
"                    if (value.script){\n"+
"                        value.script = \"true=True\\nfalse=False\\neventInfo=\"+JSON.stringify(eventInfo) + \"\\n\" + value.script;\n"+
"                    }\n"+
"                    value.execute();\n"+
"                }\n"+
"            });\n"+
"        });\n"+
"    }else{\n"+
"        console.log(\"Warning: got a pd_event with no targetDivId\", eventInfo);\n"+
"    }\n"+
"});";
    try{
        g.appendChild(document.createTextNode(code));
    }catch(e){
        g.text = code;
    }
    s.parentNode.insertBefore(g,s);
}
2017-07-30 14:30:42,394 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'showchrome': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'title': 'Evolution of Etherium Prices', 'handlerId': 'dataframe', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.table.TableDisplayMeta object at 0x7f7389d23470>
2017-07-30 14:30:44,227 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'handlerId': 'dataframe', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.table.TableDisplayMeta object at 0x7f7389d23470>
2017-07-30 14:30:55,221 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'handlerId': 'lineChart', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.chart.ChartDisplayMeta object at 0x7f738bbb9cf8>
2017-07-30 14:30:55,222 - pixiedust.display.chart.renderers - DEBUG - Found renderer: matplotlib - lineChart
2017-07-30 14:30:55,229 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - No Cache Entry found for F22C8FA53A9745C4912A876337AB8714
2017-07-30 14:31:07,585 - pixiedust.display.display.Display - DEBUG - chartOptions [{'metadata': {'min': 50, 'max': 100, 'default': 100, 'type': 'slider'}, 'name': 'chartsize', 'description': 'Chart Size'}, {'metadata': {'values': ['None', 'date'], 'default': '', 'type': 'dropdown'}, 'validate': <function clusterBy.<locals>.<lambda> at 0x7f738829f598>, 'refresh': True, 'name': 'clusterby', 'description': 'Cluster By'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'timeseries', 'description': 'Time Series'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'legend', 'description': 'Show legend'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logx', 'description': 'log scale on x'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logy', 'description': 'log scale on y'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'refresh': True, 'name': 'mpld3', 'description': 'D3 Rendering (mpld3)'}]
2017-07-30 14:31:07,586 - pixiedust.utils.template - DEBUG - Template already qualified pixiedust.display.chart.renderers.baseChartDisplay:baseChartOptionsDialogBody.html
2017-07-30 14:31:37,204 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'nostore_figureOnly': 'true', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'handlerId': 'lineChart', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.chart.ChartDisplayMeta object at 0x7f738bbb9cf8>
2017-07-30 14:31:37,210 - pixiedust.display.chart.renderers - DEBUG - Found renderer: matplotlib - lineChart
2017-07-30 14:31:37,213 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Found cache data for F22C8FA53A9745C4912A876337AB8714. Validating integrity...
2017-07-30 14:31:37,213 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Cache data not validated for key timeseries. Expected Value is false. Got true. Destroying it!...
2017-07-30 14:31:46,335 - pixiedust.display.display.Display - DEBUG - chartOptions [{'metadata': {'min': 50, 'max': 100, 'default': 100, 'type': 'slider'}, 'name': 'chartsize', 'description': 'Chart Size'}, {'metadata': {'values': ['None', 'date'], 'default': '', 'type': 'dropdown'}, 'validate': <function clusterBy.<locals>.<lambda> at 0x7f73881eb8c8>, 'refresh': True, 'name': 'clusterby', 'description': 'Cluster By'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'timeseries', 'description': 'Time Series'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'legend', 'description': 'Show legend'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logx', 'description': 'log scale on x'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logy', 'description': 'log scale on y'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'refresh': True, 'name': 'mpld3', 'description': 'D3 Rendering (mpld3)'}]
2017-07-30 14:31:46,335 - pixiedust.utils.template - DEBUG - Template already qualified pixiedust.display.chart.renderers.baseChartDisplay:baseChartOptionsDialogBody.html
2017-07-30 14:31:56,369 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'nostore_figureOnly': 'true', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'handlerId': 'lineChart', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.chart.ChartDisplayMeta object at 0x7f738bbb9cf8>
2017-07-30 14:31:56,369 - pixiedust.display.chart.renderers - DEBUG - Found renderer: matplotlib - lineChart
2017-07-30 14:31:56,370 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Found cache data for F22C8FA53A9745C4912A876337AB8714. Validating integrity...
2017-07-30 14:31:56,371 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Cache data validated for F22C8FA53A9745C4912A876337AB8714. Using it!...
2017-07-30 14:31:56,371 - pixiedust.display.display.Display - DEBUG - chartOptions [{'metadata': {'min': 50, 'max': 100, 'default': 100, 'type': 'slider'}, 'name': 'chartsize', 'description': 'Chart Size'}, {'metadata': {'values': ['None', 'date'], 'default': '', 'type': 'dropdown'}, 'validate': <function clusterBy.<locals>.<lambda> at 0x7f738850e7b8>, 'refresh': True, 'name': 'clusterby', 'description': 'Cluster By'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'timeseries', 'description': 'Time Series'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'legend', 'description': 'Show legend'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logx', 'description': 'log scale on x'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logy', 'description': 'log scale on y'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'refresh': True, 'name': 'mpld3', 'description': 'D3 Rendering (mpld3)'}]
2017-07-30 14:31:56,371 - pixiedust.utils.template - DEBUG - Template already qualified pixiedust.display.chart.renderers.baseChartDisplay:baseChartOptionsDialogBody.html
2017-07-30 14:32:01,049 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'logy': 'true', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'handlerId': 'lineChart', 'nostore_pixiedust': 'true', 'nostore_figureOnly': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.chart.ChartDisplayMeta object at 0x7f738bbb9cf8>
2017-07-30 14:32:01,049 - pixiedust.display.chart.renderers - DEBUG - Found renderer: matplotlib - lineChart
2017-07-30 14:32:01,050 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Found cache data for F22C8FA53A9745C4912A876337AB8714. Validating integrity...
2017-07-30 14:32:01,051 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Cache data validated for F22C8FA53A9745C4912A876337AB8714. Using it!...
2017-07-30 14:32:01,051 - pixiedust.display.display.Display - DEBUG - chartOptions [{'metadata': {'min': 50, 'max': 100, 'default': 100, 'type': 'slider'}, 'name': 'chartsize', 'description': 'Chart Size'}, {'metadata': {'values': ['None', 'date'], 'default': '', 'type': 'dropdown'}, 'validate': <function clusterBy.<locals>.<lambda> at 0x7f73881986a8>, 'refresh': True, 'name': 'clusterby', 'description': 'Cluster By'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'timeseries', 'description': 'Time Series'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'legend', 'description': 'Show legend'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logx', 'description': 'log scale on x'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logy', 'description': 'log scale on y'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'refresh': True, 'name': 'mpld3', 'description': 'D3 Rendering (mpld3)'}]
2017-07-30 14:32:01,052 - pixiedust.utils.template - DEBUG - Template already qualified pixiedust.display.chart.renderers.baseChartDisplay:baseChartOptionsDialogBody.html
2017-07-30 14:32:12,791 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'mpld3': 'true', 'logy': 'true', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'handlerId': 'lineChart', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.chart.ChartDisplayMeta object at 0x7f738bbb9cf8>
2017-07-30 14:32:12,792 - pixiedust.display.chart.renderers - DEBUG - Found renderer: matplotlib - lineChart
2017-07-30 14:32:12,793 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Found cache data for F22C8FA53A9745C4912A876337AB8714. Validating integrity...
2017-07-30 14:32:12,794 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Cache data validated for F22C8FA53A9745C4912A876337AB8714. Using it!...
2017-07-30 14:32:12,795 - pixiedust.display.display.Display - DEBUG - chartOptions [{'metadata': {'min': 50, 'max': 100, 'default': 100, 'type': 'slider'}, 'name': 'chartsize', 'description': 'Chart Size'}, {'metadata': {'values': ['None', 'date'], 'default': '', 'type': 'dropdown'}, 'validate': <function clusterBy.<locals>.<lambda> at 0x7f7388198f28>, 'refresh': True, 'name': 'clusterby', 'description': 'Cluster By'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'timeseries', 'description': 'Time Series'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'legend', 'description': 'Show legend'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logx', 'description': 'log scale on x'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logy', 'description': 'log scale on y'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'refresh': True, 'name': 'mpld3', 'description': 'D3 Rendering (mpld3)'}]
2017-07-30 14:32:12,795 - pixiedust.utils.template - DEBUG - Template already qualified pixiedust.display.chart.renderers.baseChartDisplay:baseChartOptionsDialogBody.html
2017-07-30 14:32:14,437 - pixiedust.display.display.Display - DEBUG - Converting to FixedLocator for mpld3
2017-07-30 14:32:14,443 - pixiedust.display.display.Display - ERROR - Unexpected error while trying to render BaseChartDisplay
Traceback (most recent call last):
  File "/opt/conda/lib/python3.5/site-packages/pixiedust/display/chart/renderers/baseChartDisplay.py", line 388, in doRender
    chartFigure = self.doRenderChart()
  File "/opt/conda/lib/python3.5/site-packages/pixiedust/display/chart/renderers/matplotlib/matplotlibBaseDisplay.py", line 189, in doRenderChart
    a.xaxis.set_major_locator(ticker.FixedLocator(locator.tick_values(vmin, vmax)))
  File "/opt/conda/lib/python3.5/site-packages/matplotlib/dates.py", line 1011, in tick_values
    return self.get_locator(vmin, vmax).tick_values(vmin, vmax)
  File "/opt/conda/lib/python3.5/site-packages/pandas/tseries/converter.py", line 229, in get_locator
    delta = relativedelta(dmax, dmin)
  File "/opt/conda/lib/python3.5/site-packages/dateutil/relativedelta.py", line 134, in __init__
    raise TypeError("relativedelta only diffs datetime/date")
TypeError: relativedelta only diffs datetime/date
2017-07-30 14:32:26,558 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'mpld3': 'false', 'logy': 'true', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'handlerId': 'lineChart', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.chart.ChartDisplayMeta object at 0x7f738bbb9cf8>
2017-07-30 14:32:26,559 - pixiedust.display.chart.renderers - DEBUG - Found renderer: matplotlib - lineChart
2017-07-30 14:32:26,559 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Found cache data for F22C8FA53A9745C4912A876337AB8714. Validating integrity...
2017-07-30 14:32:26,560 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Cache data validated for F22C8FA53A9745C4912A876337AB8714. Using it!...
2017-07-30 14:32:26,561 - pixiedust.display.display.Display - DEBUG - chartOptions [{'metadata': {'min': 50, 'max': 100, 'default': 100, 'type': 'slider'}, 'name': 'chartsize', 'description': 'Chart Size'}, {'metadata': {'values': ['None', 'date'], 'default': '', 'type': 'dropdown'}, 'validate': <function clusterBy.<locals>.<lambda> at 0x7f73881d4510>, 'refresh': True, 'name': 'clusterby', 'description': 'Cluster By'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'timeseries', 'description': 'Time Series'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'legend', 'description': 'Show legend'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logx', 'description': 'log scale on x'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logy', 'description': 'log scale on y'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'refresh': True, 'name': 'mpld3', 'description': 'D3 Rendering (mpld3)'}]
2017-07-30 14:32:26,561 - pixiedust.utils.template - DEBUG - Template already qualified pixiedust.display.chart.renderers.baseChartDisplay:baseChartOptionsDialogBody.html
2017-07-30 14:32:34,159 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'mpld3': 'false', 'logy': 'false', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'handlerId': 'lineChart', 'nostore_pixiedust': 'true', 'nostore_figureOnly': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.chart.ChartDisplayMeta object at 0x7f738bbb9cf8>
2017-07-30 14:32:34,162 - pixiedust.display.chart.renderers - DEBUG - Found renderer: matplotlib - lineChart
2017-07-30 14:32:34,163 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Found cache data for F22C8FA53A9745C4912A876337AB8714. Validating integrity...
2017-07-30 14:32:34,163 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Cache data validated for F22C8FA53A9745C4912A876337AB8714. Using it!...
2017-07-30 14:32:34,168 - pixiedust.display.display.Display - DEBUG - chartOptions [{'metadata': {'min': 50, 'max': 100, 'default': 100, 'type': 'slider'}, 'name': 'chartsize', 'description': 'Chart Size'}, {'metadata': {'values': ['None', 'date'], 'default': '', 'type': 'dropdown'}, 'validate': <function clusterBy.<locals>.<lambda> at 0x7f7388556510>, 'refresh': True, 'name': 'clusterby', 'description': 'Cluster By'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'timeseries', 'description': 'Time Series'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'legend', 'description': 'Show legend'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logx', 'description': 'log scale on x'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logy', 'description': 'log scale on y'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'refresh': True, 'name': 'mpld3', 'description': 'D3 Rendering (mpld3)'}]
2017-07-30 14:32:34,169 - pixiedust.utils.template - DEBUG - Template already qualified pixiedust.display.chart.renderers.baseChartDisplay:baseChartOptionsDialogBody.html
2017-07-30 14:32:43,868 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'mpld3': 'false', 'logy': 'false', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'legend': 'true', 'handlerId': 'lineChart', 'nostore_pixiedust': 'true', 'nostore_figureOnly': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.chart.ChartDisplayMeta object at 0x7f738bbb9cf8>
2017-07-30 14:32:43,869 - pixiedust.display.chart.renderers - DEBUG - Found renderer: matplotlib - lineChart
2017-07-30 14:32:43,870 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Found cache data for F22C8FA53A9745C4912A876337AB8714. Validating integrity...
2017-07-30 14:32:43,870 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Cache data validated for F22C8FA53A9745C4912A876337AB8714. Using it!...
2017-07-30 14:32:43,870 - pixiedust.display.display.Display - DEBUG - chartOptions [{'metadata': {'min': 50, 'max': 100, 'default': 100, 'type': 'slider'}, 'name': 'chartsize', 'description': 'Chart Size'}, {'metadata': {'values': ['None', 'date'], 'default': '', 'type': 'dropdown'}, 'validate': <function clusterBy.<locals>.<lambda> at 0x7f73881bb510>, 'refresh': True, 'name': 'clusterby', 'description': 'Cluster By'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'timeseries', 'description': 'Time Series'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'legend', 'description': 'Show legend'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logx', 'description': 'log scale on x'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logy', 'description': 'log scale on y'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'refresh': True, 'name': 'mpld3', 'description': 'D3 Rendering (mpld3)'}]
2017-07-30 14:32:43,871 - pixiedust.utils.template - DEBUG - Template already qualified pixiedust.display.chart.renderers.baseChartDisplay:baseChartOptionsDialogBody.html
2017-07-30 14:33:17,707 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'mpld3': 'false', 'logy': 'false', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'legend': 'true', 'handlerId': 'downloadFile', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.download.DownloadMeta object at 0x7f7389d23da0>
2017-07-30 14:33:39,500 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'doDownload': 'md', 'mpld3': 'false', 'logy': 'false', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'legend': 'true', 'handlerId': 'downloadFile', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'doDownloadCount': 100, 'chartsize': '71'}: <pixiedust.display.download.DownloadMeta object at 0x7f7389d23da0>
2017-07-30 14:33:39,893 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'doDownloadLink': 'True', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'doDownload': 'md', 'mpld3': 'false', 'logy': 'false', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'legend': 'true', 'handlerId': 'downloadFile', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'doDownloadCount': 100, 'chartsize': '71'}: <pixiedust.display.download.DownloadMeta object at 0x7f7389d23da0>
2017-07-30 14:33:55,598 - pixiedust.display - DEBUG - Creating a new display handler with options {'keyFields': 'epoch', 'timeseries': 'true', 'valueFields': 'usd', 'org_params': 'nostore_pixiedust', 'aggregation': 'MAX', 'nostore_bokeh': 'false', 'cell_id': 'F22C8FA53A9745C4912A876337AB8714', 'rowCount': '500', 'mpld3': 'false', 'logy': 'false', 'title': 'Evolution of Etherium Prices', 'prefix': 'e75d872a', 'logx': 'true', 'legend': 'true', 'handlerId': 'lineChart', 'nostore_pixiedust': 'true', 'nostore_cw': '1098', 'chartsize': '71'}: <pixiedust.display.chart.ChartDisplayMeta object at 0x7f738bbb9cf8>
2017-07-30 14:33:55,599 - pixiedust.display.chart.renderers - DEBUG - Found renderer: matplotlib - lineChart
2017-07-30 14:33:55,600 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Found cache data for F22C8FA53A9745C4912A876337AB8714. Validating integrity...
2017-07-30 14:33:55,600 - pixiedust.display.chart.renderers.baseChartDisplay - DEBUG - Cache data validated for F22C8FA53A9745C4912A876337AB8714. Using it!...
2017-07-30 14:33:55,613 - pixiedust.display.display.Display - DEBUG - chartOptions [{'metadata': {'min': 50, 'max': 100, 'default': 100, 'type': 'slider'}, 'name': 'chartsize', 'description': 'Chart Size'}, {'metadata': {'values': ['None', 'date'], 'default': '', 'type': 'dropdown'}, 'validate': <function clusterBy.<locals>.<lambda> at 0x7f7388556b70>, 'refresh': True, 'name': 'clusterby', 'description': 'Cluster By'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'timeseries', 'description': 'Time Series'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'legend', 'description': 'Show legend'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logx', 'description': 'log scale on x'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'name': 'logy', 'description': 'log scale on y'}, {'metadata': {'default': 'false', 'type': 'checkbox'}, 'refresh': True, 'name': 'mpld3', 'description': 'D3 Rendering (mpld3)'}]
2017-07-30 14:33:55,613 - pixiedust.utils.template - DEBUG - Template already qualified pixiedust.display.chart.renderers.baseChartDisplay:baseChartOptionsDialogBody.html


# Notebook from edeutsch/Community-Notebooks
Path: Notebooks/How_to_perform_an_ANOVA_test_in_BigQuery.ipynb

# ISB-CGC Community Notebooks

Check out more notebooks at our [Community Notebooks Repository](https://github.com/isb-cgc/Community-Notebooks)!

```
Title:   One-Way ANOVA Test in BigQuery
Author:  Lauren Hagen
Created: 2019-08-02
Purpose: Demonstrate an ANOVA test within BigQuery
URL:     https://github.com/isb-cgc/Community-Notebooks/blob/master/Notebooks/How_to_perform_an_ANOVA_test_in_BigQuery.ipynb
Notes:   This notebook was adapted from work by David L Gibbs, September 2017 Query of the Month
```
***

## Running ANOVA tests in BigQuery

In this notebook, we will cover how to do a one-way ANOVA test in BigQuery. This statistical test can be used to determine whether there is a statistically significant difference between the means of two or more independent groups. Although in this example, I’m only looking at two groups, it would not be difficult to extend this to any number of groups, assuming there is a reasonable number of samples within each group.

Consider the model yij = m + ai + eij, where yij is a continuous variable over samples j, in groups i, and ai is a constant for each group i, and eij is a gaussian error term with mean 0.

Using this model, we are describing the data as being sampled from groups, with each group having a mean value equal to m + ai. The null hypothesis is that each of the group means is the same (ie that the ai terms are zero), while the alternative hypothesis is that at least one of the ai terms is not zero.

We use the F-test to compare these two hypotheses. To compute the test statistic, we compute the within-group variation and the between-group variation. Recall that sample variance is defined as the sum of squared differences between observations and the mean, divided by the number of samples (normalized).

First, we will need to import bigquery module, authenticate ourselves, and create a client variable. For more information see ['Quick Start Guide to ISB-CGC'](https://github.com/isb-cgc/Community-Notebooks/blob/master/Notebooks/Quick_Start_Guide_to_ISB_CGC.ipynb) and alternative authentication methods can be found [here](https://googleapis.github.io/google-cloud-python/latest/core/auth.html).
_____no_output_____

# Import BigQuery Module
from google.cloud import bigquery_____no_output_____# Autheticate ourselves
!gcloud auth application-default login_____no_output_____# Create a variable for which client to use with BigQuery
project = 'your_project_number' # Replace with your project ID
if project_num == 'your_project_number':
    print('Please update the project number with your Google Cloud Project')
else:
    client = bigquery.Client(project) _____no_output_____

Let’s look at the query:
** Note: you will need to update 'your_project_number' with your project number before continuing with the notebook **_____no_output_____

# Magic command of bigquery with the project id as isb-cgc-02-0001 and create a Pandas Dataframe
# Change isb-cgc-02-0001 to your project ID
%%bigquery anova --project your_project_number
WITH
  -- using standard SQL,
  -- we'll select our cohort and gene expression
  --
  cohortExpr AS (
  SELECT
    sample_barcode,
    LOG10(normalized_count) AS expr
  FROM
    `isb-cgc.TCGA_hg19_data_v0.RNAseq_Gene_Expression_UNC_RSEM`
  WHERE
    project_short_name = 'TCGA-BRCA'
    AND HGNC_gene_symbol = 'TP53'
    AND normalized_count IS NOT NULL
    AND normalized_count > 0),
  --
  -- And we'll select the variant data for our cohort,
  -- we're going to be comparing variant types (SNP, DEL, etc)
  --
  cohortVar AS (
  SELECT
    Variant_Type,
    sample_barcode_tumor AS sample_barcode
  FROM
    `isb-cgc.TCGA_hg19_data_v0.Somatic_Mutation_MC3`
  WHERE
    SYMBOL = 'TP53' ),
  --
  -- then we join the expression and variant data using sample barcodes
  --
  cohort AS (
  SELECT
    cohortExpr.sample_barcode AS sample_barcode,
    Variant_Type AS group_name,
    expr
  FROM
    cohortExpr
  JOIN
    cohortVar
  ON
    cohortExpr.sample_barcode = cohortVar.sample_barcode ),
  --
  -- First part of the calculation, the grand mean (over everything)
  --
  grandMeanTable AS (
  SELECT
    AVG(expr) AS grand_mean
  FROM
    cohort ),
  --
  -- Then we need a mean per group, and we can get a count of samples
  -- per group.
  --
  groupMeansTable AS (
  SELECT
    AVG(expr) AS group_mean,
    group_name,
    COUNT(sample_barcode) AS n
  FROM
    cohort
  GROUP BY
    group_name),
  --
  -- To get the between-group variance
  -- we take the difference between the grand mean
  -- and the means for each group and sum over all samples
  -- ... a short cut being taking the product with n.
  -- Later we'll sum over the n_sq_diff
  --
  ssBetween AS (
  SELECT
    group_name,
    group_mean,
    grand_mean,
    n,
    n*POW(group_mean - grand_mean,2) AS n_diff_sq
  FROM
    groupMeansTable
  CROSS JOIN
    grandMeanTable ),
  --
  -- Then, to get the variance within each group
  -- we have to build a table matching up the group mean
  -- with the values for each group. So we join the group
  -- means to the values on group name. We are going to
  -- sum over this table just like ssBetween
  --
  ssWithin AS (
  SELECT
    a.group_name AS group_name,
    expr,
    group_mean,
    b.n AS n,
    POW(expr - group_mean, 2) AS s2
  FROM
    cohort a
  JOIN
    ssBetween b
  ON
    a.group_name = b.group_name ),
  --
  -- The F stat comes from a ratio, the numerator is
  -- calculated using the between group variance, and
  -- dividing by the number of groups (k) minus 1.
  --
  numerator AS (
  SELECT
    'dummy' AS dummy,
    SUM(n_diff_sq) / (count(group_name) - 1) AS mean_sq_between
  FROM
    ssBetween ),
  --
  -- The denominator of the F stat ratio is found using the
  -- variance within groups. We divide the sum of the within
  -- group variance and divide it by (n-k).
  --
  denominator AS (
  SELECT
    'dummy' AS dummy,
    COUNT(distinct(group_name)) AS k,
    COUNT(group_name) AS n,
    SUM(s2)/(COUNT(group_name)-COUNT(distinct(group_name))) AS mean_sq_within
  FROM
    ssWithin),
  --
  -- Now we're ready to calculate F!
  --
  Ftable AS (
  SELECT
    n,
    k,
    mean_sq_between,
    mean_sq_within,
    mean_sq_between / mean_sq_within AS F
  FROM
    numerator
  JOIN
    denominator
  ON
    numerator.dummy = denominator.dummy)

SELECT
  *
FROM
  Ftable_____no_output_____anova_____no_output_____

OK, so let’s check our work. Using the BRCA cohort and TP53 as our gene, we have 375 samples with a variant in this gene. We’re going to look at whether the type of variant is related to the gene expression we observe. If we just pull down the data using the ‘cohort’ subtable (as above), we can get a small data frame, which let’s us do the standard F stat table.
_____no_output_____

# Magic command of bigquery with the project id as isb-cgc-02-0001 and create a Pandas Dataframe
# Change isb-cgc-02-0001 to your project ID
%%bigquery dat --project your_project_number
WITH
  -- using standard SQL,
  -- we'll select our cohort and gene expression
  --
  cohortExpr AS (
  SELECT
    sample_barcode,
    LOG10(normalized_count) AS expr
  FROM
    `isb-cgc.TCGA_hg19_data_v0.RNAseq_Gene_Expression_UNC_RSEM`
  WHERE
    project_short_name = 'TCGA-BRCA'
    AND HGNC_gene_symbol = 'TP53'
    AND normalized_count IS NOT NULL
    AND normalized_count > 0),
  --
  -- And we'll select the variant data for our cohort,
  -- we're going to be comparing variant types (SNP, DEL, etc)
  --
  cohortVar AS (
  SELECT
    Variant_Type,
    sample_barcode_tumor AS sample_barcode
  FROM
    `isb-cgc.TCGA_hg19_data_v0.Somatic_Mutation_MC3`
  WHERE
    SYMBOL = 'TP53' ),
  --
  -- then we join the expression and variant data using sample barcodes
  --
  cohort AS (
  SELECT
    cohortExpr.sample_barcode AS sample_barcode,
    Variant_Type AS group_name,
    expr
  FROM
    cohortExpr
  JOIN
    cohortVar
  ON
    cohortExpr.sample_barcode = cohortVar.sample_barcode )
SELECT sample_barcode, group_name, expr
FROM cohort_____no_output_____dat.head()_____no_output_____import pandas as pd_____no_output_____dat.groupby(['group_name']).describe()_____no_output_____from scipy import stats_____no_output_____dat_DEL = dat[dat['group_name']=='DEL']
dat_INS = dat[dat['group_name']=='INS']
dat_SNP = dat[dat['group_name']=='SNP']
stats.f_oneway(dat_DEL['expr'],dat_INS['expr'],dat_SNP['expr'])_____no_output_____

OK, if you run the above BigQuery, you’ll see the same results. We see that the F statistic is really high, which makes sense looking at the difference in mean expression values across the groups (these are log10 expression values)._____no_output_____


# Notebook from edsaac/bioparticle
Path: notebooks/Concepts/.ipynb_checkpoints/DecayRateModel-checkpoint.ipynb

%reset -f
import numpy as np
import matplotlib.pyplot as plt

from ipywidgets import interact_manual
import ipywidgets as wd

from os import system_____no_output_____import matplotlib.text as mtext

class LegendTitle(object):
    def __init__(self, text_props=None):
        self.text_props = text_props or {}
        super(LegendTitle, self).__init__()

    def legend_artist(self, legend, orig_handle, fontsize, handlebox):
        x0, y0 = handlebox.xdescent, handlebox.ydescent
        title = mtext.Text(x0, y0, r'\bf{' + orig_handle + '}', usetex=False, **self.text_props)
        handlebox.add_artist(title)
        return title_____no_output_____

# Guillier et al. model (2020)_____no_output_____

def Guillier2020(Temp):
    '''Other Constants in the model'''
    LOG10DREF = 2.3
    TREF = 4.0
    ZT = 29.1

    log10D = lambda T : LOG10DREF - ((T - TREF)/ZT)**2
    D = np.power(10.0,log10D(Temp))

    decayRate = np.log(10.0)/D #hour^-1

    if type(Temp) is float:
        print("Guillier et al. (2020)")
        print("Time to 1-log reduction [h] = {0:.4E} ".format(D))
        print("                        [d] = {0:.4E} ".format(D/24))
        print("")
        print("Time to 6-log reduction [h] = {0:.4E} ".format(D*6))
        print("                        [d] = {0:.4E} ".format(D*6/24))

        print("decay Rate [1/h] = {0:.2E} ".format(decayRate))
        print("           [1/d] = {0:.2E} ".format(decayRate*24))
        print("           [1/s] = {0:.4E} ".format(decayRate/3600))

    return decayRate/3600_____no_output_____def Bivins2020(Temp):
    SLOPE = 0.0612
    YINTERCEPT = -0.978

    log10K = lambda T : SLOPE*T + YINTERCEPT
    decayRate = np.power(10.0,log10K(Temp)) #day^-1

    if type(Temp) is float:
        print("Bivins et al. (2020)")
        print("decay Rate [1/h] = {0:.2E} ".format(decayRate/24))
        print("           [1/d] = {0:.2E} ".format(decayRate))
        print("           [1/s] = {0:.4E} ".format(decayRate/86400))

    return decayRate/86400_____no_output_____def Silverman2020(Temp):
    SLOPE = 0.065
    YINTERCEPT = -1.66

    log10K = lambda T : SLOPE*T + YINTERCEPT
    decayRate = np.power(10.0,log10K(Temp)) #day^-1

    if type(Temp) is float:
        print("Silverman et Boehm (2020)")
        print("decay Rate [1/h] = {0:.2E} ".format(decayRate/24))
        print("           [1/d] = {0:.2E} ".format(decayRate))
        print("           [1/s] = {0:.4E} ".format(decayRate/86400))

    return decayRate/86400_____no_output_____def AhmedRNA2020(Temp):
    SLOPE = 0.016
    YINTERCEPT = -1.16

    log10K = lambda T : SLOPE*T + YINTERCEPT
    decayRate = np.power(10.0,log10K(Temp)) #day^-1

    if type(Temp) is float:
        print("Ahmed et al. -RNA- (2020)")
        print("decay Rate [1/h] = {0:.2E} ".format(decayRate/24))
        print("           [1/d] = {0:.2E} ".format(decayRate))
        print("           [1/s] = {0:.4E} ".format(decayRate/86400))

    return decayRate/86400_____no_output_____def deOliveira2021():
    T = np.array([4,24])
    decayRate = {"RiverWater" : np.array([0.16,0.37])/86400.,
                 "WasteWater" : np.array([0.19,0.83])/86400.}  #d^-1>>s^-1
    return (T,decayRate)_____no_output_____def SalaComorera2021():
    T = np.array([4,20])
    decayRate = {"RiverWater" : np.array([0.61,1.01])/86400.,
                 "SeaWater" : np.array([1.07,2.02])/86400.}  #d^-1>>s^-1
    return (T,decayRate)_____no_output_____

plt.figure(figsize=(5,5))
ax = plt.subplot(1,1,1)
lines={}

## Guillier 2020
T = np.arange(4,40,1)
lines['Guillier'] = ax.plot(T,Guillier2020(T),\
         label="Guillier et al. (2020)\n  · Coronaviruses",\
         c="#d01c8b",lw=5)

## Bivins 2020
T = np.arange(20,70,1)
T1 = np.arange(4,25,1)
lines['Bivins'] = ax.plot(T,Bivins2020(T),\
         label="\u2020 Bivins et al. (2020)\n  · SARS-CoV-2 ",\
         c="#d7191c",lw=4)
lines['Bivins_extrapol'] = ax.plot(T1,Bivins2020(T1),\
         c="#d7191c",lw=1.5,ls="dotted",alpha=0.8)

#BIVINSDATA = dict(T=np.array([20,50,70]),
#                  K=np.array([1.4,216,1440]))
#plt.scatter(BIVINSDATA['T'],BIVINSDATA['K']/86400,\
#           s=50,c='green',marker="o")

## Silverman 2020
T = np.arange(4,50,1)
lines['Silverman'] = ax.plot(T,Silverman2020(T),\
         label="Silverman et Boehm (2020)\n  · Coronaviruses",\
         c="#fdae61",lw=4)

## Ahmed 2020
T = np.arange(4,37,1)
lines['AhmedRNA'] = ax.plot(T,AhmedRNA2020(T),\
         label="Ahmed et al. (2020)\n  · SARS-CoV-2 RNA signal",\
         c="gray",lw=2,ls="dashed")

## deOliveira 2021
lines['OliveriraWW'] = ax.scatter(deOliveira2021()[0],deOliveira2021()[1]['WasteWater'],\
            label="de Oliveira et al. (2021)\n  · Wastewater",\
            c="#b8e186",marker="o",s=35,zorder=4,edgecolors='blue')

lines['OliveriraRW'] = ax.scatter(deOliveira2021()[0],deOliveira2021()[1]['RiverWater'],\
            label="  · Riverwater",\
            c="#b8e186",marker="o",s=35,zorder=4,edgecolors='black')

## Sala-Comorera 2021
lines['SalaComoreraRW'] = ax.scatter(SalaComorera2021()[0],SalaComorera2021()[1]['RiverWater'],\
            label="Sala-Comorera et al. (2021)\n  · Riverwater",\
            c="#4dac26",marker="s",s=35,zorder=4,edgecolors='black')

lines['SalaComoreraSW'] = ax.scatter(SalaComorera2021()[0],SalaComorera2021()[1]['SeaWater'],\
            label="  · Seawater",\
            c="#4dac26",marker="s",s=35,zorder=4,edgecolors='blue')

## Miscelaneous
ax.set(yscale="log",ylim=[1.0E-7,1.0E-3])
font = {'size': 12}
ax.set_ylabel("Decay rate\n"+r"$\lambda_{\rm aq}$ [s$^{-1}$]",fontdict=font)
ax.set_xlabel("Temperature\n"+r"$T$ [°C]",fontdict=font)
ax.set(xlim=[0,40],xticks=[0,10,20,30,40])

## Secondary axis
def rate2t99(rate):
    t99 = -(np.log10(1./100.)*np.log(10.0))/(rate*86400)
    return t99

def t992rate(t99):
    rate = -(np.log10(1./100.)*np.log(10.0))/(t99*86400)
    return rate

secax = ax.secondary_yaxis('right', functions=(rate2t99, t992rate))
secax.set_ylabel(r'$t_{99}$'+' [d]',rotation=-90,fontdict=font,labelpad=15.0)

## Legend
whitebox = ax.scatter([1],[1],c="white",marker="o",s=1,alpha=0)

handles = [whitebox,
           lines['Guillier'][0],
           lines['Silverman'][0],
           lines['Bivins'][0],
           whitebox,
           whitebox,
           lines['OliveriraWW'],
           lines['OliveriraRW'],
           lines['SalaComoreraRW'],
           lines['SalaComoreraSW']]

labels = ["Temperature-decay rate\nmodels",
          lines['Guillier'][0].get_label(),
          lines['Silverman'][0].get_label(),
          lines['Bivins'][0].get_label(),
          "",
          "Individual SARS-CoV-2\ndata points",
          lines['OliveriraWW'].get_label(),
          lines['OliveriraRW'].get_label(),
          lines['SalaComoreraRW'].get_label(),
          lines['SalaComoreraSW'].get_label()]

leg1 = ax.legend(handles, labels, title="Infectivity decay",
           title_fontsize="large",
           loc="center left",ncol=2,bbox_to_anchor=(1.20,0.6),\
          edgecolor='k',facecolor='mintcream',labelspacing=1)

handles2 = [lines['AhmedRNA'][0]]
labels2  = [lines['AhmedRNA'][0].get_label()]

leg2 = ax.legend(handles2, labels2,title="RNA signal decay",
           bbox_to_anchor=(1.20,0.1),
           loc="center left",ncol=1,title_fontsize="large",\
          edgecolor='k',facecolor='mintcream',labelspacing=1)

ax.add_artist(leg1)
#ax.legend(loc="center left",ncol=1,bbox_to_anchor=(1.20,0.25),\
#          edgecolor='k',facecolor='mintcream',labelspacing=1)

#plt.tight_layout()
plt.show()<ipython-input-18-77cb6b12b7ec>:68: RuntimeWarning: divide by zero encountered in true_divide
  rate = -(np.log10(1./100.)*np.log(10.0))/(t99*86400)