# Notebook from anuragdogra2192/LegoMinifiguresDetection
Path: TrainData/.ipynb_checkpoints/DataPrep-checkpoint.ipynb

<code>
'''
Data Preparation of Lego DataSet
'''_____no_output_____import pandas as pd
print(pd.__version__)1.3.4
index_df = pd.read_csv('index.csv')_____no_output_____meta_df = pd.read_csv('metadata.csv')_____no_output_____print(index_df.to_string())                            path  class_id
0            marvel/0001/001.jpg         1
1            marvel/0001/002.jpg         1
2            marvel/0001/003.jpg         1
3            marvel/0001/004.jpg         1
4            marvel/0001/005.jpg         1
5            marvel/0001/006.jpg         1
6            marvel/0001/007.jpg         1
7            marvel/0001/008.jpg         1
8            marvel/0001/009.jpg         1
9            marvel/0001/010.jpg         1
10           marvel/0002/001.jpg         2
11           marvel/0002/002.jpg         2
12           marvel/0002/003.jpg         2
13           marvel/0002/004.jpg         2
14           marvel/0002/005.jpg         2
15           marvel/0002/006.jpg         2
16           marvel/0002/007.jpg         2
17           marvel/0002/008.jpg         2
18           marvel/0002/009.jpg         2
19           marvel/0002/010.jpg         2
20           marvel/0003/001.jpg         3
21           marvel/0003/002.jpg         3
22           marvel/0003/003.jpg         3
23           marvel/0003/004.jpg         3
24           marvel/0003/005.jpg         3
25           marvel/0003/006.jpg         3
26           marvel/0003/007.jpg         3
27           marvel/0003/008.jpg         3
28           marvel/0003/009.jpg         3
29           marvel/0003/010.jpg         3
30           marvel/0004/001.jpg         4
31           marvel/0004/002.jpg         4
32           marvel/0004/003.jpg         4
33           marvel/0004/004.jpg         4
34           marvel/0004/005.jpg         4
35           marvel/0004/006.jpg         4
36           marvel/0004/007.jpg         4
37           marvel/0004/008.jpg         4
38           marvel/0004/009.jpg         4
39           marvel/0004/010.jpg         4
40        star-wars/0001/001.jpg         5
41        star-wars/0001/002.jpg         5
42        star-wars/0001/003.jpg         5
43        star-wars/0001/004.jpg         5
44        star-wars/0001/005.jpg         5
45        star-wars/0001/006.jpg         5
46        star-wars/0001/007.jpg         5
47        star-wars/0001/008.jpg         5
48        star-wars/0001/009.jpg         5
49        star-wars/0001/010.jpg         5
50        star-wars/0001/011.jpg         5
51        star-wars/0001/012.jpg         5
52        star-wars/0002/001.jpg         6
53        star-wars/0002/002.jpg         6
54        star-wars/0002/003.jpg         6
55        star-wars/0002/004.jpg         6
56        star-wars/0002/005.jpg         6
57        star-wars/0002/006.jpg         6
58        star-wars/0002/007.jpg         6
59        star-wars/0002/008.jpg         6
60        star-wars/0002/009.jpg         6
61        star-wars/0002/010.jpg         6
62        star-wars/0002/011.jpg         6
63        star-wars/0002/012.jpg         6
64        star-wars/0003/001.jpg         7
65        star-wars/0003/002.jpg         7
66        star-wars/0003/003.jpg         7
67        star-wars/0003/004.jpg         7
68        star-wars/0003/005.jpg         7
69        star-wars/0003/006.jpg         7
70        star-wars/0003/007.jpg         7
71        star-wars/0003/008.jpg         7
72        star-wars/0003/009.jpg         7
73        star-wars/0003/010.jpg         7
74        star-wars/0003/011.jpg         7
75        star-wars/0003/012.jpg         7
76        star-wars/0004/001.jpg         8
77        star-wars/0004/002.jpg         8
78        star-wars/0004/003.jpg         8
79        star-wars/0004/004.jpg         8
80        star-wars/0004/005.jpg         8
81        star-wars/0004/006.jpg         8
82        star-wars/0004/007.jpg         8
83        star-wars/0004/008.jpg         8
84        star-wars/0004/009.jpg         8
85        star-wars/0004/010.jpg         8
86        star-wars/0004/011.jpg         8
87        star-wars/0005/001.jpg         9
88        star-wars/0005/002.jpg         9
89        star-wars/0005/003.jpg         9
90        star-wars/0005/004.jpg         9
91        star-wars/0005/005.jpg         9
92        star-wars/0005/006.jpg         9
93        star-wars/0005/007.jpg         9
94        star-wars/0005/008.jpg         9
95        star-wars/0005/009.jpg         9
96        star-wars/0005/010.jpg         9
97        star-wars/0005/011.jpg         9
98        star-wars/0005/012.jpg         9
99        star-wars/0006/001.jpg        10
100       star-wars/0006/002.jpg        10
101       star-wars/0006/003.jpg        10
102       star-wars/0006/004.jpg        10
103       star-wars/0006/005.jpg        10
104       star-wars/0006/006.jpg        10
105       star-wars/0006/007.jpg        10
106       star-wars/0006/008.jpg        10
107       star-wars/0007/001.jpg        11
108       star-wars/0007/002.jpg        11
109       star-wars/0007/003.jpg        11
110       star-wars/0007/004.jpg        11
111       star-wars/0007/005.jpg        11
112       star-wars/0007/006.jpg        11
113       star-wars/0007/007.jpg        11
114       star-wars/0008/001.jpg        12
115       star-wars/0008/002.jpg        12
116       star-wars/0008/003.jpg        12
117       star-wars/0008/004.jpg        12
118       star-wars/0008/005.jpg        12
119       star-wars/0008/006.jpg        12
120       star-wars/0008/007.jpg        12
121       star-wars/0009/001.jpg        13
122       star-wars/0009/002.jpg        13
123       star-wars/0009/003.jpg        13
124       star-wars/0009/004.jpg        13
125       star-wars/0009/005.jpg        13
126       star-wars/0009/006.jpg        13
127       star-wars/0009/007.jpg        13
128       star-wars/0010/001.jpg        14
129       star-wars/0010/002.jpg        14
130       star-wars/0010/003.jpg        14
131       star-wars/0010/004.jpg        14
132       star-wars/0010/005.jpg        14
133       star-wars/0010/006.jpg        14
134       star-wars/0010/007.jpg        14
135          marvel/0005/001.jpg        15
136          marvel/0005/002.jpg        15
137          marvel/0005/003.jpg        15
138          marvel/0005/004.jpg        15
139          marvel/0005/005.jpg        15
140          marvel/0005/006.jpg        15
141          marvel/0005/007.jpg        15
142          marvel/0005/008.jpg        15
143          marvel/0006/001.jpg        16
144          marvel/0006/002.jpg        16
145          marvel/0006/003.jpg        16
146          marvel/0006/004.jpg        16
147          marvel/0006/005.jpg        16
148          marvel/0006/006.jpg        16
149          marvel/0006/007.jpg        16
150          marvel/0006/008.jpg        16
151          marvel/0007/001.jpg        17
152          marvel/0007/002.jpg        17
153          marvel/0007/003.jpg        17
154          marvel/0007/004.jpg        17
155          marvel/0007/005.jpg        17
156          marvel/0007/006.jpg        17
157          marvel/0007/007.jpg        17
158          marvel/0007/008.jpg        17
159    harry-potter/0001/001.jpg        18
160    harry-potter/0001/002.jpg        18
161    harry-potter/0001/003.jpg        18
162    harry-potter/0001/004.jpg        18
163    harry-potter/0001/005.jpg        18
164    harry-potter/0001/006.jpg        18
165    harry-potter/0001/007.jpg        18
166    harry-potter/0001/008.jpg        18
167    harry-potter/0001/009.jpg        18
168    harry-potter/0001/010.jpg        18
169    harry-potter/0001/011.jpg        18
170    harry-potter/0002/001.jpg        19
171    harry-potter/0002/002.jpg        19
172    harry-potter/0002/003.jpg        19
173    harry-potter/0002/004.jpg        19
174    harry-potter/0002/005.jpg        19
175    harry-potter/0002/006.jpg        19
176    harry-potter/0002/007.jpg        19
177    harry-potter/0002/008.jpg        19
178    harry-potter/0002/009.jpg        19
179    harry-potter/0002/010.jpg        19
180    harry-potter/0002/011.jpg        19
181          marvel/0008/001.jpg        20
182          marvel/0008/002.jpg        20
183          marvel/0008/003.jpg        20
184          marvel/0008/004.jpg        20
185          marvel/0008/005.jpg        20
186          marvel/0008/006.jpg        20
187          marvel/0008/007.jpg        20
188          marvel/0008/008.jpg        20
189          marvel/0008/009.jpg        20
190          marvel/0008/010.jpg        20
191          marvel/0008/011.jpg        20
192          marvel/0009/001.jpg        21
193          marvel/0009/002.jpg        21
194          marvel/0009/003.jpg        21
195          marvel/0009/004.jpg        21
196          marvel/0009/005.jpg        21
197          marvel/0009/006.jpg        21
198          marvel/0009/007.jpg        21
199          marvel/0009/008.jpg        21
200          marvel/0009/009.jpg        21
201          marvel/0009/010.jpg        21
202          marvel/0009/011.jpg        21
203          marvel/0010/001.jpg        22
204          marvel/0010/002.jpg        22
205          marvel/0010/003.jpg        22
206          marvel/0010/004.jpg        22
207          marvel/0010/005.jpg        22
208          marvel/0010/006.jpg        22
209          marvel/0010/007.jpg        22
210          marvel/0010/008.jpg        22
211          marvel/0010/009.jpg        22
212          marvel/0010/010.jpg        22
213          marvel/0010/011.jpg        22
214          marvel/0011/001.jpg        23
215          marvel/0011/002.jpg        23
216          marvel/0011/003.jpg        23
217          marvel/0011/004.jpg        23
218          marvel/0011/005.jpg        23
219          marvel/0011/006.jpg        23
220          marvel/0011/007.jpg        23
221          marvel/0011/008.jpg        23
222          marvel/0011/009.jpg        23
223          marvel/0011/010.jpg        23
224          marvel/0012/001.jpg        24
225          marvel/0012/002.jpg        24
226          marvel/0012/003.jpg        24
227          marvel/0012/004.jpg        24
228          marvel/0012/005.jpg        24
229          marvel/0012/006.jpg        24
230          marvel/0012/007.jpg        24
231          marvel/0012/008.jpg        24
232          marvel/0012/009.jpg        24
233          marvel/0012/010.jpg        24
234          marvel/0013/001.jpg        25
235          marvel/0013/002.jpg        25
236          marvel/0013/003.jpg        25
237          marvel/0013/004.jpg        25
238          marvel/0013/005.jpg        25
239          marvel/0013/006.jpg        25
240          marvel/0013/007.jpg        25
241          marvel/0013/008.jpg        25
242          marvel/0013/009.jpg        25
243          marvel/0013/010.jpg        25
244  jurassic-world/0001/001.jpg        26
245  jurassic-world/0001/002.jpg        26
246  jurassic-world/0001/003.jpg        26
247  jurassic-world/0001/004.jpg        26
248  jurassic-world/0001/005.jpg        26
249  jurassic-world/0001/006.jpg        26
250  jurassic-world/0001/007.jpg        26
251  jurassic-world/0001/008.jpg        26
252  jurassic-world/0001/009.jpg        26
253  jurassic-world/0001/010.jpg        26
254  jurassic-world/0001/011.jpg        26
255  jurassic-world/0001/012.jpg        26
256  jurassic-world/0002/001.jpg        27
257  jurassic-world/0002/002.jpg        27
258  jurassic-world/0002/003.jpg        27
259  jurassic-world/0002/004.jpg        27
260  jurassic-world/0002/005.jpg        27
261  jurassic-world/0002/006.jpg        27
262  jurassic-world/0002/007.jpg        27
263  jurassic-world/0002/008.jpg        27
264  jurassic-world/0002/009.jpg        27
265  jurassic-world/0002/010.jpg        27
266  jurassic-world/0002/011.jpg        27
267  jurassic-world/0002/012.jpg        27
268          marvel/0014/001.jpg        28
269          marvel/0014/002.jpg        28
270          marvel/0014/003.jpg        28
271          marvel/0014/004.jpg        28
272          marvel/0014/005.jpg        28
273          marvel/0014/006.jpg        28
274          marvel/0014/007.jpg        28
275          marvel/0014/008.jpg        28
276          marvel/0014/009.jpg        28
277          marvel/0014/010.jpg        28
278          marvel/0015/001.jpg        29
279          marvel/0015/002.jpg        29
280          marvel/0015/003.jpg        29
281          marvel/0015/004.jpg        29
282          marvel/0015/005.jpg        29
283          marvel/0015/006.jpg        29
284          marvel/0015/007.jpg        29
285          marvel/0015/008.jpg        29
286          marvel/0015/009.jpg        29
287          marvel/0015/010.jpg        29
288          marvel/0016/001.jpg        30
289          marvel/0016/002.jpg        30
290          marvel/0016/003.jpg        30
291          marvel/0016/004.jpg        30
292          marvel/0016/005.jpg        30
293          marvel/0016/006.jpg        30
294          marvel/0016/007.jpg        30
295          marvel/0016/008.jpg        30
296          marvel/0016/009.jpg        30
297          marvel/0016/010.jpg        30
298          marvel/0017/001.jpg        31
299          marvel/0017/002.jpg        31
300          marvel/0017/003.jpg        31
301          marvel/0017/004.jpg        31
302          marvel/0017/005.jpg        31
303          marvel/0017/006.jpg        31
304          marvel/0017/007.jpg        31
305          marvel/0017/008.jpg        31
306          marvel/0017/009.jpg        31
307          marvel/0017/010.jpg        31
308       star-wars/0011/001.jpg        32
309       star-wars/0011/002.jpg        32
310       star-wars/0011/003.jpg        32
311       star-wars/0011/004.jpg        32
312       star-wars/0011/005.jpg        32
313       star-wars/0011/006.jpg        32
314       star-wars/0011/007.jpg        32
315       star-wars/0011/008.jpg        32
316       star-wars/0012/001.jpg        33
317       star-wars/0012/002.jpg        33
318       star-wars/0012/003.jpg        33
319       star-wars/0012/004.jpg        33
320       star-wars/0012/005.jpg        33
321       star-wars/0012/006.jpg        33
322       star-wars/0012/007.jpg        33
323       star-wars/0012/008.jpg        33
324       star-wars/0013/001.jpg        34
325       star-wars/0013/002.jpg        34
326       star-wars/0013/003.jpg        34
327       star-wars/0013/004.jpg        34
328       star-wars/0013/005.jpg        34
329       star-wars/0013/006.jpg        34
330       star-wars/0013/007.jpg        34
331       star-wars/0013/008.jpg        34
332       star-wars/0013/009.jpg        34
333       star-wars/0014/001.jpg        35
334       star-wars/0014/002.jpg        35
335       star-wars/0014/003.jpg        35
336       star-wars/0014/004.jpg        35
337       star-wars/0014/005.jpg        35
338       star-wars/0014/006.jpg        35
339       star-wars/0014/007.jpg        35
340       star-wars/0014/008.jpg        35
341       star-wars/0014/009.jpg        35
342       star-wars/0015/001.jpg        36
343       star-wars/0015/002.jpg        36
344       star-wars/0015/003.jpg        36
345       star-wars/0015/004.jpg        36
346       star-wars/0015/005.jpg        36
347       star-wars/0015/006.jpg        36
348       star-wars/0015/007.jpg        36
349       star-wars/0015/008.jpg        36
350       star-wars/0015/009.jpg        36
351       star-wars/0015/010.jpg        36
352       star-wars/0016/001.jpg        37
353       star-wars/0016/002.jpg        37
354       star-wars/0016/003.jpg        37
355       star-wars/0016/004.jpg        37
356       star-wars/0016/005.jpg        37
357       star-wars/0016/006.jpg        37
358       star-wars/0016/007.jpg        37
359       star-wars/0016/008.jpg        37
360       star-wars/0016/009.jpg        37
print(meta_df.to_string())    class_id lego_ids                             lego_names              minifigure_name
0          1  [76115]              ['Spider Mech vs. Venom']                   SPIDER-MAN
1          2  [76115]              ['Spider Mech vs. Venom']                        VENOM
2          3  [76115]              ['Spider Mech vs. Venom']                     AUNT MAY
3          4  [76115]              ['Spider Mech vs. Venom']                 GHOST SPIDER
4          5  [75208]                         ["Yoda's Hut"]                         YODA
5          6  [75208]                         ["Yoda's Hut"]               LUKE SKYWALKER
6          7  [75208]                         ["Yoda's Hut"]                        R2-D2
7          8  [75199]   ["General Grievous' Combat Speeder"]                   MACE WINDU
8          9  [75199]   ["General Grievous' Combat Speeder"]             GENERAL GRIEVOUS
9         10  [75264]    ["Kylo Ren's Shuttle Microfighter"]                     KYLO REN
10        11  [75254]  ["AT-ST Raider from The Mandalorian"]              THE MANDALORIAN
11        12  [75254]  ["AT-ST Raider from The Mandalorian"]                    CARA DUNE
12        13  [75254]  ["AT-ST Raider from The Mandalorian"]         KLATOOINIAN RAIDER 1
13        14  [75254]  ["AT-ST Raider from The Mandalorian"]         KLATOOINIAN RAIDER 2
14        15  [76128]                  ["Molten Man Battle"]                     MYSTERIO
15        16  [76128]                  ["Molten Man Battle"]                  FIREFIGHTER
16        17  [76128]                  ["Molten Man Battle"]                   SPIDER-MAN
17        18  [75950]                      ["Aragog's Lair"]                 HARRY POTTER
18        19  [75950]                      ["Aragog's Lair"]                  RON WEASLEY
19        20  [76162]     ["Black Widow's Helicopter Chase"]                  BLACK WIDOW
20        21  [76162]     ["Black Widow's Helicopter Chase"]                YELENA BELOVA
21        22  [76162]     ["Black Widow's Helicopter Chase"]                   TASKMASTER
22        23  [76123]  ["Captain America: Outriders Attack"]              CAPTAIN AMERICA
23        24  [76123]  ["Captain America: Outriders Attack"]                   OUTRIDER 1
24        25  [76123]  ["Captain America: Outriders Attack"]                   OUTRIDER 2
25        26  [75926]                   ["Pteranodon Chase"]                   OWEN GRADY
26        27  [75926]                   ["Pteranodon Chase"]  TRACKER TRAQUEUR RASTREADOR
27        28  [76125]             ["Iron Man Hall of Armor"]                IRON MAN MK 1
28        29  [76125]             ["Iron Man Hall of Armor"]                IRON MAN MK 5
29        30  [76125]             ["Iron Man Hall of Armor"]               IRON MAN MK 41
30        31  [76125]             ["Iron Man Hall of Armor"]               IRON MAN MK 50
31        32  [75273]       ["Poe Dameron's X-wing Fighter"]                       JANNAH
32        33   [7104]                       ["Desert Skiff"]                     HAN SOLO
33        34  [75183]         ["Darth Vader Transformation"]                  DARTH VADER
34        35  [75183]         ["Darth Vader Transformation"]             ANAKIN SKYWALKER
35        36  [75183]         ["Darth Vader Transformation"]            EMPEROR PALPATINE
36        37  [75269]                   ["Duel on Mustafar"]               OBI-WAN KENOBI
i_df = index_df.drop_duplicates(subset = ["class_id"])_____no_output_____print(i_df.to_string())                            path  class_id  minifigure_name
0            marvel/0001/001.jpg         1       SPIDER-MAN
10           marvel/0002/001.jpg         2  THE MANDALORIAN
20           marvel/0003/001.jpg         3    YELENA BELOVA
30           marvel/0004/001.jpg         4   IRON MAN MK 50
40        star-wars/0001/001.jpg         5              NaN
52        star-wars/0002/001.jpg         6              NaN
64        star-wars/0003/001.jpg         7              NaN
76        star-wars/0004/001.jpg         8              NaN
87        star-wars/0005/001.jpg         9              NaN
99        star-wars/0006/001.jpg        10              NaN
107       star-wars/0007/001.jpg        11              NaN
114       star-wars/0008/001.jpg        12              NaN
121       star-wars/0009/001.jpg        13              NaN
128       star-wars/0010/001.jpg        14              NaN
135          marvel/0005/001.jpg        15              NaN
143          marvel/0006/001.jpg        16              NaN
151          marvel/0007/001.jpg        17              NaN
159    harry-potter/0001/001.jpg        18              NaN
170    harry-potter/0002/001.jpg        19              NaN
181          marvel/0008/001.jpg        20              NaN
192          marvel/0009/001.jpg        21              NaN
203          marvel/0010/001.jpg        22              NaN
214          marvel/0011/001.jpg        23              NaN
224          marvel/0012/001.jpg        24              NaN
234          marvel/0013/001.jpg        25              NaN
244  jurassic-world/0001/001.jpg        26              NaN
256  jurassic-world/0002/001.jpg        27              NaN
268          marvel/0014/001.jpg        28              NaN
278          marvel/0015/001.jpg        29              NaN
288          marvel/0016/001.jpg        30              NaN
298          marvel/0017/001.jpg        31              NaN
308       star-wars/0011/001.jpg        32              NaN
316       star-wars/0012/001.jpg        33              NaN
324       star-wars/0013/001.jpg        34              NaN
333       star-wars/0014/001.jpg        35              NaN
342       star-wars/0015/001.jpg        36              NaN
352       star-wars/0016/001.jpg        37              NaN
m_fig_name = meta_df["minifigure_name"]_____no_output_____m_fig_name_____no_output_____i_df = i_df.join(m_fig_name)_____no_output_____print(i_df)                            path  class_id  minifigure_name
0            marvel/0001/001.jpg         1       SPIDER-MAN
10           marvel/0002/001.jpg         2  THE MANDALORIAN
20           marvel/0003/001.jpg         3    YELENA BELOVA
30           marvel/0004/001.jpg         4   IRON MAN MK 50
40        star-wars/0001/001.jpg         5              NaN
52        star-wars/0002/001.jpg         6              NaN
64        star-wars/0003/001.jpg         7              NaN
76        star-wars/0004/001.jpg         8              NaN
87        star-wars/0005/001.jpg         9              NaN
99        star-wars/0006/001.jpg        10              NaN
107       star-wars/0007/001.jpg        11              NaN
114       star-wars/0008/001.jpg        12              NaN
121       star-wars/0009/001.jpg        13              NaN
128       star-wars/0010/001.jpg        14              NaN
135          marvel/0005/001.jpg        15              NaN
143          marvel/0006/001.jpg        16              NaN
151          marvel/0007/001.jpg        17              NaN
159    harry-potter/0001/001.jpg        18              NaN
170    harry-potter/0002/001.jpg        19              NaN
181          marvel/0008/001.jpg        20              NaN
192          marvel/0009/001.jpg        21              NaN
203          marvel/0010/001.jpg        22              NaN
214          marvel/0011/001.jpg        23              NaN
224          marvel/0012/001.jpg        24              NaN
234          marvel/0013/001.jpg        25              NaN
244  jurassic-world/0001/001.jpg        26              NaN
256  jurassic-world/0002/001.jpg        27              NaN
268          marvel/0014/001.jpg        28              NaN
278          marvel/0015/001.jpg        29              NaN
288          marvel/0016/001.jpg        30              NaN
298          marvel/0017/001.jpg        31              NaN
308       star-wars/0011/001.jpg        32              NaN
316       star-wars/0012/001.jpg        33              NaN
324       star-wars/0013/001.jpg        34              NaN
333       star-wars/0014/001.jpg        35              NaN
342       star-wars/0015/001.jpg        36              NaN
352       star-wars/0016/001.jpg        37              NaN

</code>


# Notebook from drewamorbordelon/SpotifySongSuggestor
Path: notebooks/PCAnalysis.ipynb

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA

from sklearn.pipeline import Pipeline
from sklearn.cluster import KMeans
from sklearn.neighbors import KDTree

import json
import requests_____no_output_____df = pd.read_csv('data/spotify-largeset.csv')
df.drop(['index'], axis=1, inplace=True)

genre_df = pd.read_csv('data/data_by_genres.csv')

genre_df['genres'].nunique()_____no_output_____genre_df.tail()_____no_output_____df['FullText'] = df['artists'].str.cat(df['name'], sep=" ")
df['FullText'] = df['FullText'].apply(lambda x: x.lower())_____no_output_____df.head(10)_____no_output_____df['FullText'][6]_____no_output_____# 3 year rolling correlation (year, accousticness)
# df.set_index('year', inplace=True, drop=True)
df.columns_____no_output_____df.drop(['explicit', 'id', 'release_date', 'popularity', 'duration_sec', 'duration_ms', 'mode'], axis=1, inplace=True)_____no_output_____

# Standardize, perform PCA, scree plot_____no_output_____

df_new = df.drop(['artists', 'name', 'Tempo Basket', 'FullText'], axis=1) #  , 'year'

x = StandardScaler().fit_transform(df_new)

# Apply PCA to the data
pca = PCA(10)
pca_features = pca.fit_transform(x)

# Define scree plot function
def scree_plot(pca):
    '''
    Creates a scree plot associated with the principal components
    INPUT: pca - the result of instantian of PCA in scikit learn
    OUTPUT:
            None
    '''
    num_components=len(pca.explained_variance_ratio_)
    ind = np.arange(num_components)
    vals = pca.explained_variance_ratio_

    plt.figure(figsize=(18, 6))
    ax = plt.subplot(111)
    cumvals = np.cumsum(vals)
    ax.bar(ind, vals)
    ax.plot(ind, cumvals)
    for i in range(num_components):
        ax.annotate(r"%s" % ((str(vals[i]*100)[:3])), (ind[i], vals[i]), va="bottom", ha="center", fontsize=4.5)

    ax.xaxis.set_tick_params(width=0)
    ax.yaxis.set_tick_params(width=2, length=10)

    ax.set_xlabel("Principal Component")
    ax.set_ylabel("Variance Explained (%)")
    plt.title('Explained Variance Per Principal Component')

scree_plot(pca)_____no_output_____# Shows how much variance is explained by each component
pca.explained_variance_ratio______no_output_____

# Kmeans with PCA_____no_output_____

pca = PCA(n_components=10)
pca.fit(df_new)

pca.transform(df_new)
scores_pca = pca.transform(df_new)
scores_pca_____no_output_____# css = cluster sum of squares
css = []
for i in range(1, 10):
    kmeans_pca = KMeans(n_clusters=i, init='k-means++', random_state=42)
    kmeans_pca.fit(scores_pca)
    css.append(kmeans_pca.inertia_)_____no_output_____

## Elbow Method_____no_output_____

plt.figure(figsize=(18, 6))
plt.plot(range(1, 10), css, marker='o', linestyle='--')
plt.xlabel('Total Clusters')
plt.ylabel("Within Cluster Sum of Squares")
plt.title("Elbow Method: K-Means with PCA Clustering")
plt.show()

print("NOTE: Elbow is at 3")_____no_output_____

# Eigen Values and Loadings_____no_output_____##  PC0 _____no_output_____

# Have a look at PC0
pc0 = pca.components_[0]
list(zip(pc0, df_new.columns))_____no_output_____eig_values = pca.explained_variance_

structure_loading_1 = pc0* np.sqrt(eig_values[0])
str_loading_1 = pd.Series(structure_loading_1, index=df_new.columns)
str_loading_1.sort_values(ascending=False)_____no_output_____

## PC1_____no_output_____

# Have a look at PC1
pc1 = pca.components_[1]
list(zip(pc1, df_new.columns))_____no_output_____eig_values = pca.explained_variance_

structure_loading_2 = pc1* np.sqrt(eig_values[1])
str_loading_2 = pd.Series(structure_loading_2, index=df_new.columns)
str_loading_2.sort_values(ascending=False)_____no_output_____

## PC3_____no_output_____

# Have a look at PC3
pc3 = pca.components_[4]
list(zip(pc3, df_new.columns))_____no_output_____eig_values = pca.explained_variance_

structure_loading_3 = pc1* np.sqrt(eig_values[9])
str_loading_3 = pd.Series(structure_loading_3, index=df_new.columns)
str_loading_3.sort_values(ascending=False)_____no_output_____df.head(1)_____no_output_____df = pd.read_csv('data/spotify-largeset.csv')
df['FullText'] = df['artists'].str.cat(df['name'], sep=" ")
df['FullText'] = df['FullText'].apply(lambda x: x.lower())

df_new = df.copy()
df_numerics = df_new.drop(['Tempo Basket', 'year', 'name',
                       'artists', 'explicit', 'release_date', 'id',
                       'popularity', 'duration_sec', 'duration_ms', 'FullText'], axis=1)

df_scaled = StandardScaler().fit_transform(df_numerics)

# Apply the model
tree = KDTree(df_scaled)_____no_output_____from joblib import dump, load

model_path = 'kd_treemodel.joblib'
dump(tree, model_path)_____no_output_____# Query the KDTree model for k nearest neighbors.
# (Alternatively) we can use query_radius
# Check sklearn docs
# https://scikit-learn.org/stable/modules/generated/
# sklearn.neighbors.KDTree.html#sklearn.neighbors.KDTree

num_of_songs = 10
dist, indxs = tree.query(df_scaled, k=num_of_songs+1)_____no_output_____

## Check distance of k=10 similar songs_____no_output_____

# Create column names first
column_names = ['Searched_Song']
for i in range(1, num_of_songs+1):
    column_names.append(f'Nearest_Song{i}')

# Calculate scores from distances
scores = (1 - ((dist - dist.min()) / (dist.max() - dist.min()))) * 100
dist_df = pd.DataFrame(scores.tolist(), columns=column_names)

# Limit decimal place output to 1
dist_df = dist_df.round(decimals=1)

# Replace Searched Song score with the track_id
dist_df['Searched_Song'] = df.FullText

# Print the shape of the dataframe
print(f'df.shape = {dist_df.shape}')

# Check distances of similar top 10 nearest songs
dist_df.head()df.shape = (165755, 11)

## Save Similar Songs_____no_output_____

# Making an array of the Track IDs
tracks = np.array(df.FullText)

# A function that creates list of the each song with its nearest neighbors
def find_similars(tracks, indxs):
    """For each track in the dataset find similar songs"""
    similars = []
    for row in indxs:
        ids = [tracks[i] for i in row]
        similars.append(ids)
    return similars

# The first item in each row is a song in the original database.
# The next 10 rows are its closest neighbors in the KDTree.
nearest_neighbors = find_similars(tracks, indxs)

# Convert it to dataframe
nearest_neighbors_df = pd.DataFrame(nearest_neighbors, columns=column_names)
nn = nearest_neighbors_df.copy()_____no_output_____final_df = nearest_neighbors_df.copy()
final_df.shape_____no_output_____# nearest_neighbors_df.query('Searched_Song == "miley cyrus adore you"')_____no_output_____# nearest_neighbors_df['Searched_Song'].filter(like='miley cyrus adore you')
nn.query('Searched_Song.str.contains("just the way")', engine='python')_____no_output_____nn['Searched_Song'][44910]_____no_output_____nearest_neighbors_df.query('Searched_Song.str.contains("greeneries")', engine='python')_____no_output_____

## Save as a .csv_____no_output_____

import sys

def save_dataframe_as_csv(df=None, filename=None):
    """
    Saves dataframe in csv format

    Arguments
    ---------
    df: Pandas DataFrame
    filename: File path or filename

    Returns
    -------
    .csv file
    """
    try:
        if not filename.endswith(".csv"):
            filenameb += ".csv"
        df.to_csv(filename, index=False)
        print(f"DataFrame saved @: {filename}")
    except Exception as e:
        print("DatFrame could not be saved: ", sys.exc_info()[0])
        raise_____no_output_____save_dataframe_as_csv(nearest_neighbors_df, "data/artist_song_df.csv")DataFrame saved @: data/artist_song_df.csv
nearest_neighbors_df.head()_____no_output_____

## Save as JSON file_____no_output_____

def save_dataframe_as_json(df=None, filename=None, orient="records"):
    """
    Saves dataframe in a JSON format

    Arguments
    ---------
    df: Pandas DataFrame
    filename: file path or filename

    Returns
    -------
    JSON file
    """
    try:
        if not filename.endswith('.json'):
            filename += '.json'
        df.to_json(filename, orient=orient)
        print(f"DataFrame saved @: {filename}")
    except Exception as e:
        print("DataFrame could not be saved: ", sys.exc_info()[0])
        raise_____no_output_____save_dataframe_as_json(nearest_neighbors_df, "data/artist_song_df.json")DataFrame saved @: data/artist_song_df.json
json_df=pd.read_json('data/artist_song_df.json')
json_df.head()_____no_output_____

## Save as a SQLite Database_____no_output_____

def save_as_sqlite_database(df, sql_uri, tableName, index=True, index_label=None, verbose=True):
    """
    Saves DataFrame as a SQLite Databse

    Arguments
    ---------
    df: Pandas DataFrame
    filename: Filepath or name

    Returns
    -------
    db.sqlite3
    """

    try:
        if not sql_uri.endswith('.sqlite3'):
            sql_uri =+ '.sqlite3'
        if not sql_uri.startswith("sqlite:///"):
            raise ValueError("Database URI should start with 'sqlite:///'")

        from sqlalchemy import create_engine
        engine = create_engine(sql_uri)
        print(f"DataFrame saved @:{sql_uri[10:]}")
        df.to_sql(tableName, con=engine, index=index, index_label=index_label, if_exists='replace')

        if verbose:
            sql = f"SELECT * FROM {tableName} LIMIT 10;"
            print(pd.read_sql(sql, con=engine))
    except Exception as e:
            print("DataFrame could not be saved: ", sys.exc_info()[0])
            raise_____no_output_____sql_uri = "sqlite:///data/db.sqlite3"
tableName = "suggestions_artist_song"
save_as_sqlite_database(final_df,
                        sql_uri,
                        tableName,
                        index=True,
                        index_label=None,
                        verbose=True)DataFrame saved @:data/db.sqlite3
   index                                      Searched_Song  \
0      0               mamie smith keep a song in your soul
1      1         screamin' jay hawkins i put a spell on you
2      2                           mamie smith golfing papa
3      3  oscar velazquez true house music - xavier sant...
4      4                                     mixe xuniverxe
5      5  mamie smith & her jazz hounds crazy blues - 78...
6      6           mamie smith don't you advertise your man
7      7       mamie smith & her jazz hounds arkansas blues
8      8      francisco canaro la chacarera - remasterizado
9      9                meetya broken puppet - original mix

                                       Nearest_Song1  \
0            billie holiday did i remember? - take 1
1  don charles', 'the singing dogs jingle bells -...
2                     p. suribabu entha ghorapathame
3                        shungi music crew the storm
4  surf mesa', 'emilee ily (i love you baby) (fea...
5                               artie shaw star dust
6                bessie smith empty bed blues, pt. 2
7                  anita o'day that's what you think
8  edvard grieg', 'andrew davis', 'philharmonia o...
9    saxon wheels of steel - 2009 remastered version

                                       Nearest_Song2  \
0            billie holiday did i remember? - take 1
1           el gran combo de puerto rico ojos chinos
2                  mamie smith the lure of the south
3          suffused', 'alfoa year 2008 - alfoa remix
4  sia', 'alan walker move your body - alan walke...
5              shaila devi ami purab desher puronari
6                bessie smith empty bed blues, pt. 1
7         kanika banerjee khelaghar bandhte legechhi
8                        tara bai ae chand na itrana
9                      slusnik luna sun 2011 - mixed

                        Nearest_Song3  \
0          peggy lee my little cousin
1                frankie ruiz la cura
2             félix mayol lilas blanc
3                jeremy olander araoz
4   the pussycat dolls when i grow up
5           art tatum st. louis blues
6          bessie smith me and my gin
7  val rosing try a little tenderness
8   oscar peterson strike up the band
9                      ac/dc sin city

                                       Nearest_Song4  \
0  louis armstrong & his hot seven', 'louis armst...
1                          jerry rivera cara de niño
2  original broadway cast of 'three wishes for ja...
3  sean & xander shimmer of tides (year in review...
4                 ariana grande no tears left to cry
5  franz liszt', 'eugene ormandy', 'philadelphia ...
6                         muddy waters rollin' stone
7  richard himber and his orchestra love thy neig...
8  wolfgang amadeus mozart', 'claudio arrau piano...
9                w&w arena [asot 384] - original mix

                                       Nearest_Song5  \
0  billie holiday', 'eddie heywood let's do it (w...
1                           the pied pipers should i
2                     tin tan y marcelo la cantinera
3  suffused', 'airwave', 'michael & levan', 'stiv...
4  aaron smith', 'krono', 'luvli dancin (feat. lu...
5              lucille bogan i'm gonna shave you dry
6  robert johnson when you got a good friend - ta...
7  dean martin money burns a hole in my pocket - ...
8       francisco canaro el baccarat - remasterizado
9  bt', 'matt fax 1 am in paris (year in review 2...

                                       Nearest_Song6  \
0  billie holiday', 'eddie heywood let's do it (w...
1                              marcos valle estrelar
2                    the pied pipers limehouse blues
3  arnej the ones that get away [asot 384] - orig...
4                    black eyed peas meet me halfway
5            francesco albanese la sagra di giarabub
6  billie holiday', 'teddy wilson it's a sin to t...
7  mohantara talpade byakul hai panghat pe radha ...
8      ignacio corsini fin de fiesta - remasterizado
9                suffused year 2008 - timewave remix

                                       Nearest_Song7  \
0           billie holiday me, myself and i - take 2
1                       willie nelson shotgun willie
2             dick haymes can i come in for a second
3  oscar velazquez true house music - original ma...
4   calvin harris', 'disciples how deep is your love
5                                    organiko ntoina
6                mariano e caçula jorginho do sertão
7                          κώστας δούσιας η τσακπίνα
8  francisco canaro yo tuve un cariño - remasteri...
9                                the prodigy breathe

                                       Nearest_Song8  \
0  robert johnson preachin' blues (up jumped the ...
1     maná mariposa traicionera - 2020 remasterizado
2                                  georgius ma gosse
3                   suffused year 2008 - alfoa remix
4                  little junior parker love my baby
5                              mixalis thomakos mama
6  billie holiday', 'teddy wilson it's a sin to t...
7                           bobby vinton blue velvet
8         mrinal kanti ghosh nishi paban nishi paban
9  mark gamble', 'mad dog no angles - feel it clu...

                                       Nearest_Song9  \
0  billie holiday', 'eddie heywood let's do it (w...
1                johnny cash the wreck of the old 97
2                         manna dey japo ram ram ram
3        arkham knights xposed (year in review 2020)
4    juice wrld let me know (i wonder why freestyle)
5  hans christian lumbye', 'orchestra of the roya...
6  billie holiday', 'teddy wilson i can't believe...
7                          billie holiday am i blue?
8  wolfgang amadeus mozart', 'maryla jonas piano ...
9            matt fax the gate (year in review 2020)

                                      Nearest_Song10
0     bing crosby i've got plenty to be thankful for
1  bob wills & his texas playboys that's what i l...
2            billie holiday you're just a no account
3                         kjrock happy new year 2017
4             new edition i'm still in love with you
5                  mohammed rafi yeh mera prem patra
6  billie holiday', 'teddy wilson i can't believe...
7                          billie holiday am i blue?
8         ignacio corsini noche mala - remasterizado
9  waveshaper', 'oscillian the year is 2003 (osci...


# Notebook from vcelis/python-3-bootcamp
Path: notebooks/12-Final Capstone Python Project/02-Final Capstone Project Ideas.ipynb

# List of Capstone Projects

This list contains around 100 project ideas for you to try out in Python! Some of them are straightforward and others we have done before (such as a Fibonacci Sequence or FizzBuzz).

Pick a simple project that you think you can finish in a day to start off with, then pick another project that you think will take you more than a week and extensive Googling!

There are some sample solutions in this folder as well so feel free to explore and remember:

**HAVE FUN!**_____no_output_____Numbers
---------

**Find PI to the Nth Digit** - Enter a number and have the program generate π (pi) up to that many decimal places. Keep a limit to how far the program will go.

**Find e to the Nth Digit** - Just like the previous problem, but with e instead of π (pi). Enter a number and have the program generate e up to that many decimal places. Keep a limit to how far the program will go.

**Fibonacci Sequence** - Enter a number and have the program generate the Fibonacci sequence to that number or to the Nth number.

**Prime Factorization** - Have the user enter a number and find all Prime Factors (if there are any) and display them.

**Next Prime Number** - Have the program find prime numbers until the user chooses to stop asking for the next one.

**Find Cost of Tile to Cover W x H Floor** - Calculate the total cost of tile it would take to cover a floor plan of width and height, using a cost entered by the user.

**Mortgage Calculator** - Calculate the monthly payments of a fixed term mortgage over given Nth terms at a given interest rate. Also figure out how long it will take the user to pay back the loan. For added complexity, add an option for users to select the compounding interval (Monthly, Weekly, Daily, Continually).

**Change Return Program** - The user enters a cost and then the amount of money given. The program will figure out the change and the number of quarters, dimes, nickels, pennies needed for the change.

**Binary to Decimal and Back Converter** - Develop a converter to convert a decimal number to binary or a binary number to its decimal equivalent.

**Calculator** - A simple calculator to do basic operators. Make it a scientific calculator for added complexity.

**Unit Converter (temp, currency, volume, mass and more)** - Converts various units between one another. The user enters the type of unit being entered, the type of unit they want to convert to and then the value. The program will then make the conversion.

**Alarm Clock** - A simple clock where it plays a sound after X number of minutes/seconds or at a particular time.

**Distance Between Two Cities** - Calculates the distance between two cities and allows the user to specify a unit of distance. This program may require finding coordinates for the cities like latitude and longitude.

**Credit Card Validator** - Takes in a credit card number from a common credit card vendor (Visa, MasterCard, American Express, Discoverer) and validates it to make sure that it is a valid number (look into how credit cards use a checksum).

**Tax Calculator** - Asks the user to enter a cost and either a country or state tax. It then returns the tax plus the total cost with tax.

**Factorial Finder** - The Factorial of a positive integer, n, is defined as the product of the sequence n, n-1, n-2, ...1 and the factorial of zero, 0, is defined as being 1. Solve this using both loops and recursion.

**Complex Number Algebra** - Show addition, multiplication, negation, and inversion of complex numbers in separate functions. (Subtraction and division operations can be made with pairs of these operations.) Print the results for each operation tested.

**Happy Numbers** - A happy number is defined by the following process. Starting with any positive integer, replace the number by the sum of the squares of its digits, and repeat the process until the number equals 1 (where it will stay), or it loops endlessly in a cycle which does not include 1. Those numbers for which this process ends in 1 are happy numbers, while those that do not end in 1 are unhappy numbers. Display an example of your output here. Find first 8 happy numbers.

**Number Names** - Show how to spell out a number in English. You can use a preexisting implementation or roll your own, but you should support inputs up to at least one million (or the maximum value of your language's default bounded integer type, if that's less). *Optional: Support for inputs other than positive integers (like zero, negative integers, and floating-point numbers).*

**Coin Flip Simulation** - Write some code that simulates flipping a single coin however many times the user decides. The code should record the outcomes and count the number of tails and heads.

**Limit Calculator** - Ask the user to enter f(x) and the limit value, then return the value of the limit statement *Optional: Make the calculator capable of supporting infinite limits.*

**Fast Exponentiation** - Ask the user to enter 2 integers a and b and output a^b (i.e. pow(a,b)) in O(lg n) time complexity.

Classic Algorithms
-----------------

**Collatz Conjecture** - Start with a number *n > 1*. Find the number of steps it takes to reach one using the following process: If *n* is even, divide it by 2. If *n* is odd, multiply it by 3 and add 1.

**Sorting** - Implement two types of sorting algorithms: Merge sort and bubble sort.

**Closest pair problem** - The closest pair of points problem or closest pair problem is a problem of computational geometry: given *n* points in metric space, find a pair of points with the smallest distance between them.

**Sieve of Eratosthenes** - The sieve of Eratosthenes is one of the most efficient ways to find all of the smaller primes (below 10 million or so).

Graph
--------

**Graph from links** - Create a program that will create a graph or network from a series of links.

**Eulerian Path** - Create a program which will take as an input a graph and output either a Eulerian path or a Eulerian cycle, or state that it is not possible.  A Eulerian Path starts at one node and traverses every edge of a graph  through every node and finishes at another node.  A Eulerian cycle is a eulerian Path that starts and finishes at the same node.

**Connected Graph** - Create a program which takes a graph as an input and outputs whether every node is connected or not.

**Dijkstra’s Algorithm** - Create a program that finds the shortest path through a graph using its edges.

**Minimum Spanning Tree** - Create a program which takes a connected, undirected graph with weights and outputs the minimum spanning tree of the graph i.e., a
subgraph that is a tree, contains all the vertices, and the sum of its weights is the least possible.

Data Structures
---------

**Inverted index** - An [Inverted Index](http://en.wikipedia.org/wiki/Inverted_index) is a data structure used to create full text search. Given a set of text files, implement a program to create an inverted index. Also create a user interface to do a search using that inverted index which returns a list of files that contain the query term / terms. The search index can be in memory.

Text
---------

**Fizz Buzz** - Write a program that prints the numbers from 1 to 100. But for multiples of three print “Fizz” instead of the number and for the multiples of five print “Buzz”. For numbers which are multiples of both three and five print “FizzBuzz”.

**Reverse a String** - Enter a string and the program will reverse it and print it out.

**Pig Latin** - Pig Latin is a game of alterations played on the English language game. To create the Pig Latin form of an English word the initial consonant sound is transposed to the end of the word and an ay is affixed (Ex.: "banana" would yield anana-bay). Read Wikipedia for more information on rules.

**Count Vowels** - Enter a string and the program counts the number of vowels in the text. For added complexity have it report a sum of each vowel found.

**Check if Palindrome** - Checks if the string entered by the user is a palindrome. That is that it reads the same forwards as backwards like “racecar”

**Count Words in a String** - Counts the number of individual words in a string. For added complexity read these strings in from a text file and generate a summary.

**Text Editor** - Notepad style application that can open, edit, and save text documents. *Optional: Add syntax highlighting and other features.*

**RSS Feed Creator** - Given a link to RSS/Atom Feed, get all posts and display them.

**Quote Tracker (market symbols etc)** - A program which can go out and check the current value of stocks for a list of symbols entered by the user. The user can set how often the stocks are checked. For CLI, show whether the stock has moved up or down. *Optional: If GUI, the program can show green up and red down arrows to show which direction the stock value has moved.*

**Guestbook / Journal** - A simple application that allows people to add comments or write journal entries. It can allow comments or not and timestamps for all entries. Could also be made into a shout box. *Optional: Deploy it on Google App Engine or Heroku or any other PaaS (if possible, of course).*

**Vigenere / Vernam / Ceasar Ciphers** - Functions for encrypting and decrypting data messages. Then send them to a friend.

**Regex Query Tool** - A tool that allows the user to enter a text string and then in a separate control enter a regex pattern. It will run the regular expression against the source text and return any matches or flag errors in the regular expression.

Networking
---------

**FTP Program** - A file transfer program which can transfer files back and forth from a remote web sever.

**Bandwidth Monitor** - A small utility program that tracks how much data you have uploaded and downloaded from the net during the course of your current online session. See if you can find out what periods of the day you use more and less and generate a report or graph that shows it.

**Port Scanner** - Enter an IP address and a port range where the program will then attempt to find open ports on the given computer by connecting to each of them. On any successful connections mark the port as open.

**Mail Checker (POP3 / IMAP)** - The user enters various account information include web server and IP, protocol type (POP3 or IMAP) and the application will check for email at a given interval.

**Country from IP Lookup** - Enter an IP address and find the country that IP is registered in. *Optional: Find the Ip automatically.*

**Whois Search Tool** - Enter an IP or host address and have it look it up through whois and return the results to you.

**Site Checker with Time Scheduling** - An application that attempts to connect to a website or server every so many minutes or a given time and check if it is up. If it is down, it will notify you by email or by posting a notice on screen.

Classes
---------

**Product Inventory Project** - Create an application which manages an inventory of products. Create a product class which has a price, id, and quantity on hand. Then create an *inventory* class which keeps track of various products and can sum up the inventory value.

**Airline / Hotel Reservation System** - Create a reservation system which books airline seats or hotel rooms. It charges various rates for particular sections of the plane or hotel. Example, first class is going to cost more than coach. Hotel rooms have penthouse suites which cost more. Keep track of when rooms will be available and can be scheduled.

**Company Manager** - Create an hierarchy of classes - abstract class Employee and subclasses HourlyEmployee, SalariedEmployee, Manager and Executive. Every one's pay is calculated differently, research a bit about it.
After you've established an employee hierarchy, create a Company class that allows you to manage the employees. You should be able to hire, fire and raise employees.

**Bank Account Manager** - Create a class called Account which will be an abstract class for three other classes called CheckingAccount, SavingsAccount and BusinessAccount. Manage credits and debits from these accounts through an ATM style program.

**Patient / Doctor Scheduler** - Create a patient class and a doctor class. Have a doctor that can handle multiple patients and setup a scheduling program where a doctor can only handle 16 patients during an 8 hr work day.

**Recipe Creator and Manager** - Create a recipe class with ingredients and a put them in a recipe manager program that organizes them into categories like deserts, main courses or by ingredients like chicken, beef, soups, pies etc.

**Image Gallery** - Create an image abstract class and then a class that inherits from it for each image type. Put them in a program which displays them in a gallery style format for viewing.

**Shape Area and Perimeter Classes** - Create an abstract class called Shape and then inherit from it other shapes like diamond, rectangle, circle, triangle etc. Then have each class override the area and perimeter functionality to handle each shape type.

**Flower Shop Ordering To Go** - Create a flower shop application which deals in flower objects and use those flower objects in a bouquet object which can then be sold. Keep track of the number of objects and when you may need to order more.

**Family Tree Creator** - Create a class called Person which will have a name, when they were born and when (and if) they died. Allow the user to create these Person classes and put them into a family tree structure. Print out the tree to the screen.

Threading
---------

**Create A Progress Bar for Downloads** - Create a progress bar for applications that can keep track of a download in progress. The progress bar will be on a separate thread and will communicate with the main thread using delegates.

**Bulk Thumbnail Creator** - Picture processing can take a bit of time for some transformations. Especially if the image is large. Create an image program which can take hundreds of images and converts them to a specified size in the background thread while you do other things. For added complexity, have one thread handling re-sizing, have another bulk renaming of thumbnails etc.

Web
---------

**Page Scraper** - Create an application which connects to a site and pulls out all links, or images, and saves them to a list. *Optional: Organize the indexed content and don’t allow duplicates. Have it put the results into an easily searchable index file.*

**Online White Board** - Create an application which allows you to draw pictures, write notes and use various colors to flesh out ideas for projects. *Optional: Add feature to invite friends to collaborate on a white board online.*

**Get Atomic Time from Internet Clock** - This program will get the true atomic time from an atomic time clock on the Internet. Use any one of the atomic clocks returned by a simple Google search.

**Fetch Current Weather** - Get the current weather for a given zip/postal code. *Optional: Try locating the user automatically.*

**Scheduled Auto Login and Action** - Make an application which logs into a given site on a schedule and invokes a certain action and then logs out. This can be useful for checking web mail, posting regular content, or getting info for other applications and saving it to your computer.

**E-Card Generator** - Make a site that allows people to generate their own little e-cards and send them to other people. Do not use Flash. Use a picture library and perhaps insightful mottos or quotes.

**Content Management System** - Create a content management system (CMS) like Joomla, Drupal, PHP Nuke etc. Start small. *Optional: Allow for the addition of modules/addons.*

**Web Board (Forum)** - Create a forum for you and your buddies to post, administer and share thoughts and ideas.

**CAPTCHA Maker** - Ever see those images with letters a numbers when you signup for a service and then asks you to enter what you see? It keeps web bots from automatically signing up and spamming. Try creating one yourself for online forms.

Files
---------

**Quiz Maker** - Make an application which takes various questions from a file, picked randomly, and puts together a quiz for students. Each quiz can be different and then reads a key to grade the quizzes.

**Sort Excel/CSV File Utility** - Reads a file of records, sorts them, and then writes them back to the file. Allow the user to choose various sort style and sorting based on a particular field.

**Create Zip File Maker** - The user enters various files from different directories and the program zips them up into a zip file. *Optional: Apply actual compression to the files. Start with Huffman Algorithm.*

**PDF Generator** - An application which can read in a text file, html file or some other file and generates a PDF file out of it. Great for a web based service where the user uploads the file and the program returns a PDF of the file. *Optional: Deploy on GAE or Heroku if possible.*

**Mp3 Tagger** - Modify and add ID3v1 tags to MP3 files. See if you can also add in the album art into the MP3 file’s header as well as other ID3v2 tags.

**Code Snippet Manager** - Another utility program that allows coders to put in functions, classes or other tidbits to save for use later. Organized by the type of snippet or language the coder can quickly look up code. *Optional: For extra practice try adding syntax highlighting based on the language.*

Databases
---------

**SQL Query Analyzer** - A utility application which a user can enter a query and have it run against a local database and look for ways to make it more efficient.

**Remote SQL Tool** - A utility that can execute queries on remote servers from your local computer across the Internet. It should take in a remote host, user name and password, run the query and return the results.

**Report Generator** - Create a utility that generates a report based on some tables in a database. Generates a sales reports based on the order/order details tables or sums up the days current database activity.

**Event Scheduler and Calendar** - Make an application which allows the user to enter a date and time of an event, event notes and then schedule those events on a calendar. The user can then browse the calendar or search the calendar for specific events. *Optional: Allow the application to create re-occurrence events that reoccur every day, week, month, year etc.*

**Budget Tracker** - Write an application that keeps track of a household’s budget. The user can add expenses, income, and recurring costs to find out how much they are saving or losing over a period of time. *Optional: Allow the user to specify a date range and see the net flow of money in and out of the house budget for that time period.*

**TV Show Tracker** - Got a favorite show you don’t want to miss? Don’t have a PVR or want to be able to find the show to then PVR it later? Make an application which can search various online TV Guide sites, locate the shows/times/channels and add them to a database application. The database/website then can send you email reminders that a show is about to start and which channel it will be on.

**Travel Planner System** - Make a system that allows users to put together their own little travel itinerary and keep track of the airline / hotel arrangements, points of interest, budget and schedule.

Graphics and Multimedia
---------

**Slide Show** - Make an application that shows various pictures in a slide show format. *Optional: Try adding various effects like fade in/out, star wipe and window blinds transitions.*

**Stream Video from Online** - Try to create your own online streaming video player.

**Mp3 Player** - A simple program for playing your favorite music files. Add features you think are missing from your favorite music player.

**Watermarking Application** - Have some pictures you want copyright protected? Add your own logo or text lightly across the background so that no one can simply steal your graphics off your site. Make a program that will add this watermark to the picture. *Optional: Use threading to process multiple images simultaneously.*

**Turtle Graphics** - This is a common project where you create a floor of 20 x 20 squares. Using various commands you tell a turtle to draw a line on the floor. You have move forward, left or right, lift or drop pen etc. Do a search online for "Turtle Graphics" for more information. *Optional: Allow the program to read in the list of commands from a file.*

**GIF Creator** A program that puts together multiple images (PNGs, JPGs, TIFFs) to make a smooth GIF that can be exported. *Optional: Make the program convert small video files to GIFs as well.*

Security
-------------

**Caesar cipher** - Implement a Caesar cipher, both encoding and decoding. The key is an integer from 1 to 25. This cipher rotates the letters of the alphabet (A to Z). The encoding replaces each letter with the 1st to 25th next letter in the alphabet (wrapping Z to A). So key 2 encrypts "HI" to "JK", but key 20 encrypts "HI" to "BC". This simple "monoalphabetic substitution cipher" provides almost no security, because an attacker who has the encoded message can either use frequency analysis to guess the key, or just try all 25 keys_____no_output_____


# Notebook from cohmathonc/biosci670
Path: IntroductionComputationalMethods/exercises_solutions/07_ODEs_solution.ipynb

_____no_output_____

import numpy as np
import matplotlib.pylab as plt_____no_output_____

# Solving ODEs_____no_output_____You see the following plot (if not displayed, view [here](https://github.com/cohmathonc/biosci670/blob/master/IntroductionComputationalMethods/exercises/figs/ODE_oscillating_solution.png)) in a report:

The authors state that the plot shows the simulated time evolution of a specific cell growth model, and they report an oscillatory growth pattern. They also mention that the explicit Euler method was used to solve the ODE that describes the growth model.

1. You suspect that the reported oscillatory pattern might be a numerical artifact. Why?

   (Review *1.1.1 Euler Method (explicit)* in the [ODE notebook](https://github.com/cohmathonc/biosci670/blob/master/IntroductionComputationalMethods/05_IntroCompMethods_SolvingODEs.ipynb))

2. How could the authors test numerically whether the pattern that they discovered is indeed a feature of their ODE model?

3. After contacting the authors, you learned that their ODE is based on the *logistic growth model*
$$\frac{dy}{dt}=r\,y\, (1-\frac{y}{K}) \, ,\tag{1}$$
with parameter choices $r=0.1$, $K=1$and initial value $y_0=0.1$.

  Approximate $y(t)$ numerically in the range from $t_0=0$ and $t_\text{max}=100$ using the explicit Euler method. Solve the model for 3 different step sizes $\Delta t = \{10, 5, 1\}$. What are the implications for question (1)?

4. (**optional**) As in question (3) but now use the Runge-Kutta RK4 method with fixed time step (see [notebook](https://github.com/cohmathonc/biosci670/blob/master/IntroductionComputationalMethods/05_IntroCompMethods_SolvingODEs.ipynb)) for solving the ODE.

5. (**optional**) Use the RK45 method with adaptive time stepping provided by `scipy.integrate` through the `solve_ivp` interface.
   The solution object contains time steps and estimated function values resulting from adaptive time stepping.

6. (**optional**) Read the description of the `rtol` option of `scipy.integrate.solve_ivp`. How do different choices of `rtol` to affect the size/number of time-steps computed by the adaptive algorithm.

_____no_output_____## Solution_____no_output_____### Possible explanations for oscillatory pattern_____no_output_____The oscillatory pattern could be a sign of numeric instability:
- Authors mention that explicit Euler method was used. This method requires small time steps to ensure convergence to the real solution and may become unstable otherwise.
- No precise information about the stepsize is provided, but the plot indicates that very large stepsizes $\Delta t > 10$ have been used for integration.

See example in exercise (2) of section *1.1.1 Euler Method (explicit)* in the [ODE notebook](https://github.com/cohmathonc/biosci670/blob/master/IntroductionComputationalMethods/05_IntroCompMethods_SolvingODEs.ipynb).
_____no_output_____### Numeric tests to confirm results_____no_output_____The authors could repeat their simulation using a different integration method, and/or decrease the step size of their simulation.
If the oscillatory pattern cannot be reproduced with finer step sizes, it is likely an artifact of the numerical method for the specific ODE and parameters used for its integration._____no_output_____### Solve with explicit Euler_____no_output_____

# from lecture notes L05
# https://github.com/cohmathonc/biosci670/blob/master/IntroductionComputationalMethods/L05_IntroCompMethods_SolvingODEs.ipynb

def log_growth(t, y):
    return 0.1*y*(1-y)

def solve_euler(f, t, y_0):
    """
    Uses explicit euler method to solve ODE: y'=f(t, y)
    with initial value y(t_0)=y_0.

    Args:
    - f: Function object that computes y', expected signature f(t, y),
         where t is evaluation point and y is function value
    - t: array of evaluation points
    - y_0: initial value, i.e. y(t[0])

    Returns:
    - array containing approximated function values y(t[i])
    """
    y = np.zeros(t.shape)
    y[0] = y_0
    for i in range(0, len(t)-1):
        y[i+1] = y[i] + (t[i+1]-t[i]) * f(t[i],y[i])
    return y_____no_output_____t_min = 0
t_max = 100
y_0 = 0.1

fig = plt.figure(figsize=plt.figaspect(0.5))
ax = fig.add_subplot(111)

for delta_t in [10, 5, 1]:
    t = np.arange(t_min, t_max, delta_t)
    y_euler_exp = solve_euler(log_growth, t, y_0)
    # plot
    ax.plot(t,y_euler_exp, label='explicit Euler $\Delta t$ = %.2f'%delta_t)

ax.set_ylabel("normalized cell density N")
ax.set_xlabel('time')
ax.set_title('simulated cell density')
ax.legend()_____no_output_____

Decreasing the stepsize results in a smooth non-oscillating curve that stabilizes at N=1 for large t.

Larger stepsizes result in oscillatory patterns around the carrying capacity $K$, $N=1$. This becomes more evident when extending the integration interval:_____no_output_____

t_min = 0
t_max = 200
y_0 = 0.1

fig = plt.figure(figsize=plt.figaspect(0.5))
ax = fig.add_subplot(111)

for delta_t in [10, 15, 20, 25]:
    t = np.arange(t_min, t_max, delta_t)
    y_euler_exp = solve_euler(log_growth, t, y_0)
    # plot
    ax.plot(t,y_euler_exp, label='explicit Euler $\Delta t$ = %.2f'%delta_t)

ax.set_ylabel("normalized cell density N")
ax.set_xlabel('time')
ax.set_title('simulated cell density')
ax.legend()_____no_output_____

The oscillatory pattern presented in the question is indeed a numeric artifact.

However,  the oscillatory pattern shown in fig 1 cannot be reproduced with the parameters specified in Q3. This indicates that numerical stability depends on the ode and its parameterization as well as the integration stepsize (and also that different parameters have been used to produce the plot)._____no_output_____### RK4 (optional)_____no_output_____

# from lecture notes L05
# https://github.com/cohmathonc/biosci670/blob/master/IntroductionComputationalMethods/L05_IntroCompMethods_SolvingODEs.ipynb

def solve_RK4(f, t, y_0):
    """
    Uses 4-step Runge-Kutta method to solve ODE: y'=f(t, y)
    with initial value y(t_0)=y_0.

    Args:
    - f: Function object that computes y', expected signature f(t, y),
         where t is evaluation point and y is function value
    - t: array of evaluation points
    - y_0: initial value, i.e. y(t[0])

    Returns:
    - array containing approximated function values y(t[i])
    """
    y = np.zeros(t.shape)
    y[0] = y_0
    for i in range(0, len(t)-1):
        delta_t = t[i+1]-t[i]
        k_1  = delta_t * f(t[i], y[i])
        k_2  = delta_t * f(t[i] + delta_t/2, y[i] + k_1/2)
        k_3  = delta_t * f(t[i] + delta_t/2, y[i] + k_2/2)
        k_4  = delta_t * f(t[i] + delta_t, y[i] + k_3)
        y[i+1] = y[i] + 1/6*(k_1 + 2*k_2 + 2*k_3 + k_4)
    return y

_____no_output_____t_min = 0
t_max = 100
y_0 = 0.1

fig = plt.figure(figsize=plt.figaspect(0.5))
ax = fig.add_subplot(111)

for delta_t in [10, 5, 1]:
    t = np.arange(t_min, t_max, delta_t)
    y_euler_exp = solve_RK4(log_growth, t, y_0)
    ax.plot(t,y_euler_exp, label='RK4 $\Delta t$ = %.2f'%delta_t)

ax.set_ylabel("normalized cell density N")
ax.set_xlabel('time')
ax.set_title('simulated cell density')
ax.legend()_____no_output_____

The approximation error of this method for a timestep of given size is smaller than the approximation error made by the Euler method.

This method is also less prone to the instabilities observed in Q3 for the Euler method.  _____no_output_____### RK45 from `scipy.integrate` (optional)_____no_output_____

# using RK45 solver via scipy.integrate.solve_ivp also shown in L05
# https://github.com/cohmathonc/biosci670/blob/master/IntroductionComputationalMethods/L05_IntroCompMethods_SolvingODEs.ipynb

from scipy.integrate import solve_ivp

t_min = 0
t_max = 200
y_0 = 0.1

fig = plt.figure(figsize=plt.figaspect(0.5))
ax = fig.add_subplot(111)
sol = solve_ivp(log_growth, [t_min, t_max], [y_0],
                method='RK45', rtol=1E-6)
# plot
ax.plot(sol.t,sol.y[0], label='RK45 (scipy.integrate)',
       marker='x')
ax.set_ylabel("normalized cell density N")
ax.set_xlabel('time')
ax.set_title('simulated cell density')
ax.legend()_____no_output_____

###  Tolerance in adaptive time stepping (optional)_____no_output_____The `rtol` option allows to modify the maximum *relative error* that is considered acceptable in each timestep. Decreasing `rtol` to smaller values means that the solution algorithm tries to find more accurate approximations which will translate into smaller step sizes. In turn, increasing `rtol` means that less accurate solutions are acceptable, which implies that the stepsize can be increased._____no_output_____###### About
This notebook is part of the *biosci670* course on *Mathematical Modeling and Methods for Biomedical Science*.
See https://github.com/cohmathonc/biosci670 for more information and material._____no_output_____