# Notebook from SYNALEAN/Mycode
Path: correlation research/readFile.ipynb

<code>
import os
import gzip
import pandas as pd

path = "E:/课题组相关/科创II/RNA-seq/"
files= os.listdir(path)
path2 = 'E:/课题组相关/科创II/RNA-seq-calculated/'
Exonanno = pd.read_csv('geneid_anno_1.csv',header=0)
Exonanno.rename(columns={'1': 'exonlen'}, inplace=True)
def getfilename(filepath):
    fileName = os.listdir(filepath)
    maxlength = 0
    maxindex = 0
    for index in range(0,len(fileName)):
        thisname = fileName[index]
        if maxlength <= len(thisname):
            maxlength = len(thisname)
            maxindex = index

    return fileName[maxindex]_____no_output_____for i in range(0,len(files)):
    test = files[i]
    testpath = path + test
    fileName = getfilename(testpath)
    DF = pd.read_csv(testpath+'/'+fileName, compression='gzip',sep='\t',header=None)
    DF.rename(columns={0: 'loadedanno.gene_id'}, inplace=True)
    merged = pd.merge(DF,Exonanno[['loadedanno.gene_id','exonlen']])
    merged.dropna(axis=0,how='any')
    rperL = merged[1]/merged['exonlen'] #这里可能出错？
    fenmu = rperL.sum()
    tpm = rperL/fenmu*1000000
    resultedDF = pd.concat([merged['loadedanno.gene_id'], tpm], axis=1)
    resultedDF.rename(columns={0: 'TPM'}, inplace=True)
    resultedDF.to_csv(path2+fileName,index=0) #虽然叫这个名字但是确实是csv格式？
    print(fileName + '_finished')
    print(str(i))
02759563-fa2b-4d96-ab78-dd7fee7513d4.htseq.counts.gz_finished
0
6520f1eb-78e5-4459-b25b-62b5d6696bc0.htseq.counts.gz_finished
1
57680511-b7a1-48d8-a79f-32d0a340c501.htseq.counts.gz_finished
2
167b3fae-0a99-4202-94ef-c83648f54fa9.htseq.counts.gz_finished
3
63691c59-d630-4b06-82bb-3d0587638a33.htseq.counts.gz_finished
4
04dc4da1-1d1a-46da-a9d8-da9964591aec.htseq.counts.gz_finished
5
3a04fb30-378c-48a0-a02e-85cf81b31b43.htseq.counts.gz_finished
6
4dcd18d6-b7b2-4a92-bcbe-361c7278e84e.htseq.counts.gz_finished
7
8b9e3751-9d6b-43cf-be29-f9c11a583b17.htseq.counts.gz_finished
8
e8201ee9-2e0a-4a8e-bc99-6e2482da919d.htseq.counts.gz_finished
9
10a32136-8a4d-4af3-8e70-543b2a336c91.htseq.counts.gz_finished
10
15864159-be88-41c8-bdef-c2c5927cb1a1.htseq.counts.gz_finished
11
a865bc94-a60d-4927-a842-8c0d6fced029.htseq.counts.gz_finished
12
21d669f3-9822-49d9-b7cd-cc0a8b955ae9.htseq.counts.gz_finished
13
16a67a95-996d-47d0-a311-6dfda6eba085.htseq.counts.gz_finished
14
f7182de7-067e-467b-af88-790ad3e63eec.htseq.counts.gz_finished
15
5260addc-0596-42c0-85c6-b74744db9cda.htseq.counts.gz_finished
16
ca5d0721-99cf-4b7c-b708-6acea2ae86bd.htseq.counts.gz_finished
17
2d61b6b3-ba40-48e8-a881-d0d73fd94c1a.htseq.counts.gz_finished
18
0cae8943-749f-4314-aad0-b7f33e78c6e1.htseq.counts.gz_finished
19
d8586777-f71e-433a-bdbb-2245fd5e2864.htseq.counts.gz_finished
20
aead190d-5225-42df-b11e-696de98d8011.htseq.counts.gz_finished
21
1d4e6a0a-f24c-42dc-9bb4-f788dd1ea183.htseq.counts.gz_finished
22
8b6f4f28-7113-4e15-8ea4-a5561e3cb145.htseq.counts.gz_finished
23
dfb1ab26-8751-4b67-9edc-2c8c9bf15f76.htseq.counts.gz_finished
24
5c0f9cb8-cc53-49ce-8384-6acb54cb1855.htseq.counts.gz_finished
25
f6893b7f-2472-416b-b137-33dae673d150.htseq.counts.gz_finished
26
c17b3f1c-06e7-4c0b-b4c6-3585caa69999.htseq.counts.gz_finished
27
5f004b7a-b777-4039-9309-8e1f128ff637.htseq.counts.gz_finished
28
d464fbfd-a937-41d5-ae6c-e4ad47b25c07.htseq.counts.gz_finished
29
f5e50380-3507-4002-9bfe-624bc0a3eba1.htseq.counts.gz_finished
30
88522aae-be2d-4899-b32a-c35a39c25b89.htseq.counts.gz_finished
31
293bd050-b485-496e-9b2c-02564e790879.htseq.counts.gz_finished
32
623e5b23-675d-4f04-84e1-67a7a1569381.htseq.counts.gz_finished
33
fc1963bb-e709-467b-950f-0177512ba35e.htseq.counts.gz_finished
34
58957ce0-4efa-4194-a3e6-37984ac010c7.htseq.counts.gz_finished
35
31a7a535-c256-4c8e-bbef-597571b134d6.htseq.counts.gz_finished
36
427eda57-677d-4879-8e78-9c7f1c9cd9a1.htseq.counts.gz_finished
37
643b7f6a-3bc1-46bd-8ca7-c127e04c0563.htseq.counts.gz_finished
38
7a0818ac-0f38-4e44-87da-55d812e447cd.htseq.counts.gz_finished
39
124e781f-fb13-4e9c-9feb-8df19c83fd86.htseq.counts.gz_finished
40
916bd9c7-c01b-4ba5-923a-c501769fb5fa.htseq.counts.gz_finished
41
4e461640-5d1c-4fd9-92ff-c5d79af0069d.htseq.counts.gz_finished
42
c1c1111e-858a-438e-8bb6-90df4defcede.htseq.counts.gz_finished
43
5af9d74c-549b-4a44-8918-b80b4bc6fab4.htseq.counts.gz_finished
44
8c9a74f3-0562-4d9d-a0e6-27a151d5a534.htseq.counts.gz_finished
45
fa2eed0c-4ee1-43fd-8a81-a0bd65f1d2c4.htseq.counts.gz_finished
46
5e2f16e9-0720-4688-9e95-9fc480c8ade7.htseq.counts.gz_finished
47
92a2f2ea-717f-42a9-80d6-fbc90ab7134c.htseq.counts.gz_finished
48
70a755a4-250e-4b5c-a341-a63f1ff430ae.htseq.counts.gz_finished
49
9eb9e10b-30ff-44db-a0e9-06fb53e33641.htseq.counts.gz_finished
50
c31622fb-6640-4050-8867-b695fe2d6a79.htseq.counts.gz_finished
51
2950c137-45e0-458f-987f-31de7f7befc4.htseq.counts.gz_finished
52
d6706bff-1f56-43a5-a180-1223034cfc09.htseq.counts.gz_finished
53
cfe24c0a-1d21-49bb-8279-838a4f059e8a.htseq.counts.gz_finished
54
f5d259d3-a462-4d0f-919a-c0783801dc5c.htseq.counts.gz_finished
55
d3c28191-633f-4e3b-b22d-d5cd4d8686fa.htseq.counts.gz_finished
56
0224b65f-ac08-4f17-b22b-e17bff4d25f4.htseq.counts.gz_finished
57
ad1b1ba2-a050-4d62-9e44-4dda1b68aa2e.htseq.counts.gz_finished
58
c79e2e9a-9749-4b7f-9ef4-0b893d818b73.htseq.counts.gz_finished
59
ebe5dd61-df1f-43a1-b85f-24ce0d8c0d02.htseq.counts.gz_finished
60
42ceca43-2a1a-4f8a-baf7-c19adce95173.htseq.counts.gz_finished
61
8f676c67-2594-4be6-897e-03f356f46cc5.htseq.counts.gz_finished
62
50a313a9-99f6-4ca0-84b3-0f4b59ae0a1a.htseq.counts.gz_finished
63
99d0fd68-6617-4a7a-af2b-a5f8eecc42b8.htseq.counts.gz_finished
64
08ac28e4-a7e8-4693-b877-52794fa8ae44.htseq.counts.gz_finished
65
7b77e8cb-3f1d-4af7-b609-d75f912484c4.htseq.counts.gz_finished
66
5792bef7-7189-4be0-8c31-1e17a95504b8.htseq.counts.gz_finished
67
b9882fe3-d5a7-4563-9380-33b958f81bb2.htseq.counts.gz_finished
68
87fb2f83-5e75-4891-ac3d-09fb20e32b09.htseq.counts.gz_finished
69
e6bb29b7-b632-4e61-bc17-ef2e9173289b.htseq.counts.gz_finished
70
51055281-028a-442a-aff8-829dc01f8114.htseq.counts.gz_finished
71
7d6e7cb6-598a-4723-8ea1-60fee87edafe.htseq.counts.gz_finished
72
b72fdad0-4062-4320-b451-08302e4ef12d.htseq.counts.gz_finished
73
ba57cbaf-b52c-402b-b72a-f0df0e086369.htseq.counts.gz_finished
74
0bd46112-1b21-4d87-97b4-6967213036f3.htseq.counts.gz_finished
75
c68e4fc6-a690-4667-9335-af2eb563a8b5.htseq.counts.gz_finished
76
9f88f85d-3c23-4389-b488-f0b69f651301.htseq.counts.gz_finished
77
b0775fe5-a9f4-4be4-bd7d-37acfd5cc631.htseq.counts.gz_finished
78
fda300eb-a8da-4665-bd1f-acf6e67bcaf4.htseq.counts.gz_finished
79
604c5918-95f3-47e4-96b0-b94c791fcd27.htseq.counts.gz_finished
80
3254c984-f9ec-4db5-9add-94489e000a62.htseq.counts.gz_finished
81
1e9b3641-f9a3-4e6b-bd2b-4f5eadb44a95.htseq.counts.gz_finished
82
527459cd-e8a6-44ce-850c-4fe2346059a7.htseq.counts.gz_finished
83
799b5a2a-e001-49c2-bf0e-167b84a2309d.htseq.counts.gz_finished
84
aafeade1-c999-487f-9a76-582f33545eee.htseq.counts.gz_finished
85
39340015-dfac-4372-a7e5-e68236727c62.htseq.counts.gz_finished
86
678d7cdd-0a63-41b2-9203-fa1385494399.htseq.counts.gz_finished
87
881bb503-95c6-4a88-9267-da8174ce4350.htseq.counts.gz_finished
88
ca023ee2-7396-4c9b-a4cb-56853200a605.htseq.counts.gz_finished
89
fe964742-d9b9-4e82-bf49-6fff3e14df70.htseq.counts.gz_finished
90
966ad64a-5a6d-4219-ac8e-adbfb9a79cad.htseq.counts.gz_finished
91
3ef61128-ac08-4452-b21c-17004a3b4632.htseq.counts.gz_finished
92
68056e09-70f8-470e-bb3a-08fe178c705e.htseq.counts.gz_finished
93
13918b1d-75f3-4195-83f2-cf64ab4359b9.htseq.counts.gz_finished
94
22bc2908-b3ef-40fd-a652-b3384a79bc43.htseq.counts.gz_finished
95
007d074e-3b7c-484b-b757-1a60e1172506.htseq.counts.gz_finished
96
1ca44e12-b515-4d1a-8835-26df82e58104.htseq.counts.gz_finished
97
fe3596cb-7fed-41b9-ae30-caebcf717e6e.htseq.counts.gz_finished
98
1c261173-3a2b-4eed-abce-692e9b3f15fe.htseq.counts.gz_finished
99
609420eb-0650-4187-8b48-cb1ea5fa0d8d.htseq.counts.gz_finished
100
c56a53fd-2141-4531-8c4d-d782038cbb7d.htseq.counts.gz_finished
101
a86552ef-96e7-4cb8-9c94-157f57fc6694.htseq.counts.gz_finished
102
0ea05ccb-3752-491c-acc2-ee397fef52da.htseq.counts.gz_finished
103
b6a2c03a-c8ad-41e9-8a19-8f5ac53cae9f.htseq.counts.gz_finished
104
ece0fe94-ea0b-4d38-bd30-a6bf9c1f3367.htseq.counts.gz_finished
105
8111546f-1eec-4ae9-a1f2-0a1548b2f61d.htseq.counts.gz_finished
106
75fbb2a1-84b0-420a-a988-87c5726ca7db.htseq.counts.gz_finished
107
78847e00-15b8-45a5-8b5a-db379f48a1c1.htseq.counts.gz_finished
108
b44cdb13-1812-4bf6-825e-edae5bc5376a.htseq.counts.gz_finished
109
dfb41de0-fd76-4884-aa76-5b0677bfeb7c.htseq.counts.gz_finished
110
32841ed1-16e3-4a88-8f5c-7aaf24c84e70.htseq.counts.gz_finished
111
fa43e80b-8b46-43d6-aaed-389cc4d535f6.htseq.counts.gz_finished
112
4d5b0ba8-64d8-404b-9a83-fc4111686afe.htseq.counts.gz_finished
113
1e775cd6-ae23-4bfa-8696-72ce922ccf4b.htseq.counts.gz_finished
114
5326afe8-db1e-4c54-9d65-c0dfe4df5e62.htseq.counts.gz_finished
115
b5631f88-a4cd-4ec6-b0e9-30512aa38728.htseq.counts.gz_finished
116
28973531-fd03-4f86-8cf5-4c6941560c37.htseq.counts.gz_finished
117
46c0216c-ff17-4ebf-ad8f-b054542fc2a9.htseq.counts.gz_finished
118
2a6ae2e8-733f-4349-8eec-d25e2b4abf86.htseq.counts.gz_finished
119
aa1e3af8-db36-45a8-b15b-0b1ff726dcd3.htseq.counts.gz_finished
120
bc5738b1-7d71-4c40-969f-68616ed01345.htseq.counts.gz_finished
121
5a0dee52-1a55-42ff-94de-cdad38802e73.htseq.counts.gz_finished
122
bab0168d-a91c-4977-a3bc-6958a0e49473.htseq.counts.gz_finished
123
77c7bf1b-e986-4e9c-832d-a64e063c54ff.htseq.counts.gz_finished
124
f7b18d67-1e4e-46d3-8f70-e40863e56a39.htseq.counts.gz_finished
125
6b59b355-7581-426d-98c1-36e132a77c4e.htseq.counts.gz_finished
126
e3067466-096f-4bea-993f-b0469a7e11e3.htseq.counts.gz_finished
127
2dad895e-fde7-4526-9af8-e69c23df9afa.htseq.counts.gz_finished
128
f27747c6-ceb3-4849-a30a-9eb6229a5576.htseq.counts.gz_finished
129
03bab812-d67c-4dae-93e0-55346525fc99.htseq.counts.gz_finished
130
096e442a-b0ce-4d28-b901-a84e0d752ec4.htseq.counts.gz_finished
131
65e649e9-860f-466a-8ebd-7cc5b4d332fb.htseq.counts.gz_finished
132
b3b6981a-b5db-481c-bc27-76a114856b03.htseq.counts.gz_finished
133
43b969ef-d7cb-4c8d-b6de-fdfa48b8edff.htseq.counts.gz_finished
134
c4e074f2-e21d-4a1a-acf3-b7916f11fb43.htseq.counts.gz_finished
135
54e697d0-a54b-465e-a2b2-96008c428056.htseq.counts.gz_finished
136
38ed53de-d5bc-4c94-8f5d-c2b8e8e65782.htseq.counts.gz_finished
137
cd052da2-9e32-4881-87c0-5e66cdc6f5f7.htseq.counts.gz_finished
138
eaa01a14-f563-4076-8696-9064c9397e6a.htseq.counts.gz_finished
139
fd90487c-4f23-4270-a54c-e97ebe5f03c8.htseq.counts.gz_finished
140
4f5d1a22-65de-4a73-9d31-86693aeced90.htseq.counts.gz_finished
141
a9b9f900-b114-48b4-95c7-41793107169f.htseq.counts.gz_finished
142
19e1f123-a36f-4580-a255-35b95ac0a7df.htseq.counts.gz_finished
143
339d6e55-7065-45d5-ba2c-afc689cb7386.htseq.counts.gz_finished
144
f7fb7de2-e995-4e5b-9362-b4e20ed44070.htseq.counts.gz_finished
145
70057cae-a8cb-441a-96d3-9f08002c8f9b.htseq.counts.gz_finished
146
f87f2bab-c823-41dc-aac7-75d626ec8200.htseq.counts.gz_finished
147
f38b2f21-c5a5-4699-9727-eae9c96aac55.htseq.counts.gz_finished
148
dd8003eb-e16a-424a-9092-520676779c53.htseq.counts.gz_finished
149
ede52976-0412-4878-a28e-1b1b39316e0f.htseq.counts.gz_finished
150
09c38ee5-b39e-47b2-8f27-8f125affa7f6.htseq.counts.gz_finished
151
c633bd52-3f48-49b8-bb20-2aad7b910a47.htseq.counts.gz_finished
152
f440449e-a22f-49db-9206-9e615bc1dfc0.htseq.counts.gz_finished
153
d3925cdd-7ca1-4c69-89b2-f51b15297f5e.htseq.counts.gz_finished
154
f0ce3674-e566-474f-8b8e-96c467e3d7f5.htseq.counts.gz_finished
155
31024f61-9191-4a91-a8fd-a8f4b63e6da6.htseq.counts.gz_finished
156
2e410838-f351-4645-bf70-d3018e6e2963.htseq.counts.gz_finished
157
b6a39fab-afc2-4498-bb4c-95adeb388c6f.htseq.counts.gz_finished
158
7489c557-4c73-48d4-8fbb-3b5d029cf8a9.htseq.counts.gz_finished
159
8677d42c-aef1-444d-b9a2-0dbcced0e886.htseq.counts.gz_finished
160
fa3d5856-e25b-4d4e-94ac-f524dc8a4cf5.htseq.counts.gz_finished
161
04e84524-f75f-4cb5-9e9c-e78850d7b6f5.htseq.counts.gz_finished
162
0cad7bff-30bd-4c4a-accf-876422f3cb6a.htseq.counts.gz_finished
163
4ebd9936-0b27-4aae-ad0e-7ef56cea61c9.htseq.counts.gz_finished
164
36c6972a-3041-478c-a68d-702af9cc4954.htseq.counts.gz_finished
165
5a7c65fc-a8d4-4f1e-8c76-7bb937c327ca.htseq.counts.gz_finished
166
61c7c0b0-d730-4d67-b1b7-c267124d4e1b.htseq.counts.gz_finished
167
97d0fb11-7826-46c6-95bc-5ad8fa082490.htseq.counts.gz_finished
168
cbf61237-f5ba-4fa0-97be-e79e8e75a887.htseq.counts.gz_finished
169
23293089-6e36-4347-98c4-c11a9e33c6b5.htseq.counts.gz_finished
170
1c860457-48e6-4a74-a455-e61bc10c6b32.htseq.counts.gz_finished
171
03248d19-cb6a-4578-9759-c0c4de048920.htseq.counts.gz_finished
172
55d3a539-99e2-4643-9ce4-96c01cfc6781.htseq.counts.gz_finished
173
7d33e8dc-bb77-4448-8ef4-b4b6e1d67206.htseq.counts.gz_finished
174
82e00e45-734c-471f-ba97-79ec3b7e0baa.htseq.counts.gz_finished
175
3ceffeff-4dd6-4ff9-8fa2-eb88199c498b.htseq.counts.gz_finished
176
e70fb08e-1239-4035-874e-64dbb3ac0e3d.htseq.counts.gz_finished
177
96b44e28-67a2-4781-a85b-c82fe528157d.htseq.counts.gz_finished
178
21b1c050-5bb4-44a1-8fcc-4819d6f14c01.htseq.counts.gz_finished
179
060ae2b3-e1ae-41e4-9b67-ba3938f90d3a.htseq.counts.gz_finished
180
aa42dd76-efb1-4fb7-9922-cf54540844e7.htseq.counts.gz_finished
181
ca7c56ab-9248-4c27-8992-8f73746d8d9b.htseq.counts.gz_finished
182
f15cc8f3-bd23-47d0-8401-3e9ba2fe1f46.htseq.counts.gz_finished
183
a059cf08-5046-4a79-a82f-c07da2955035.htseq.counts.gz_finished
184
3893a3ce-8554-4de1-9070-4ad443e4fa03.htseq.counts.gz_finished
185
c670b26e-b107-4279-bdd1-7ed502023f45.htseq.counts.gz_finished
186
21e82bf2-237f-4f8f-86d0-aaf2ea5c4729.htseq.counts.gz_finished
187
b4acb23a-4f0f-40dd-be0e-2faa2f196f9f.htseq.counts.gz_finished
188
1afc81dd-cbac-439f-a65e-2ccd07516812.htseq.counts.gz_finished
189
71857763-8175-4a80-8bf4-ebf3c5018151.htseq.counts.gz_finished
190
b4219e05-aeeb-430d-9310-e6f1541e5ef1.htseq.counts.gz_finished
191
47f0749e-01c7-4d8c-8926-06293c4a9b4e.htseq.counts.gz_finished
192
38c31608-d1ad-4eb2-96f9-924f6a8d3219.htseq.counts.gz_finished
193
0733fd68-9cd5-440a-a849-fb14ef76c00d.htseq.counts.gz_finished
194
20ff57b5-eeb2-4312-a5e0-11ac8c8d573a.htseq.counts.gz_finished
195
a0c56eec-568a-46b0-88db-f14d64a3942b.htseq.counts.gz_finished
196
8eb179a6-a0af-4628-aae2-78800db3e9d8.htseq.counts.gz_finished
197
aa47670a-4c3f-468a-b2d3-9d7b1d60845e.htseq.counts.gz_finished
198
32340e45-00fa-4739-be97-3934c30cb1f6.htseq.counts.gz_finished
199
cf680d44-e1de-496c-b547-12a42cf831a3.htseq.counts.gz_finished
200
d5ed2249-5c62-4590-a758-2f4e7dc46998.htseq.counts.gz_finished
201
a8ecb5d8-ece5-43cc-843c-da81dc031ced.htseq.counts.gz_finished
202
59c0ac33-df0f-44d5-8cbe-9fe90c82b9d9.htseq.counts.gz_finished
203
456bf061-467d-4bd8-bf71-b0a54ef06dea.htseq.counts.gz_finished
204
bf92fe23-899b-4dd6-96bb-182c53d7f18d.htseq.counts.gz_finished
205
46f8e9d8-032c-471e-876c-1af0ac3b1cee.htseq.counts.gz_finished
206
c49c187f-98e7-463d-a18b-accf3205eb81.htseq.counts.gz_finished
207
bf78430f-6c70-4e76-ade6-a434e0770d5f.htseq.counts.gz_finished
208
7e56863a-53f2-4c94-9cf0-053d72b4d5ee.htseq.counts.gz_finished
209
8a677c7e-9e03-4510-9c7e-b52311f144d9.htseq.counts.gz_finished
210
0563a8a4-fff0-449e-840a-888af6180d6c.htseq.counts.gz_finished
211
950e2ba0-a247-4bd6-8092-f97cc4018a79.htseq.counts.gz_finished
212
b1868d9d-ef06-4353-883b-230dae23f8b6.htseq.counts.gz_finished
213
c0606a63-53f9-40d4-bdfe-f1fc74c26609.htseq.counts.gz_finished
214
01fc4b6a-e68d-4a6f-b472-75d1620f4172.htseq.counts.gz_finished
215
f790ef4c-71bf-43a7-a610-4006a4c0a42c.htseq.counts.gz_finished
216
bfe796aa-7f29-49d2-a72f-79a9acf12365.htseq.counts.gz_finished
217
84492608-6c8f-4a40-a057-4fba4614fdfa.htseq.counts.gz_finished
218
43b292be-5d63-4523-a43f-666d20039208.htseq.counts.gz_finished
219
71d0b85c-4b4e-45b8-af0c-ca264d05df4c.htseq.counts.gz_finished
220
6c9e0596-c697-4d5e-a97f-1a0b42970c36.htseq.counts.gz_finished
221
f442826c-7516-45f7-bbfc-968cd4ce51b3.htseq.counts.gz_finished
222
a5e07df6-2166-422f-b852-dc5da2e3a876.htseq.counts.gz_finished
223
8e913f54-6511-4dba-9406-23b9eb2feec1.htseq.counts.gz_finished
224
ef53264e-cf1b-417d-a985-c54ff4ef6026.htseq.counts.gz_finished
225
676bff6f-27b7-492c-87fd-ef23dc3d23fc.htseq.counts.gz_finished
226
d6478e3f-149e-4b99-82c2-182405f1040b.htseq.counts.gz_finished
227
eab5b6fd-7f4d-41a8-83cb-5b09161b95cc.htseq.counts.gz_finished
228
c06751c4-ac35-4da4-a86d-36b245de9c16.htseq.counts.gz_finished
229
ba282b05-9ddd-448a-bbb8-49c3b98c8284.htseq.counts.gz_finished
230
61f2d13b-a11b-48e2-ab4b-84ca27d54eb2.htseq.counts.gz_finished
231
f60b4be3-9340-4e44-9e5b-f840a1bd9f9c.htseq.counts.gz_finished
232
ec78640c-f4de-4d18-bf7c-9e5391feb915.htseq.counts.gz_finished
233
996000bc-080d-4559-80bf-c5059838622a.htseq.counts.gz_finished
234
591b1beb-d902-4d96-822c-66ca24693bba.htseq.counts.gz_finished
235
42e6517c-02ff-491b-8315-9ef163e6cfdc.htseq.counts.gz_finished
236
984ea13e-adc6-48f7-830d-66ce450a2ea0.htseq.counts.gz_finished
237
72fe3744-0461-4ea9-b03d-f75929eafc51.htseq.counts.gz_finished
238
eff731a9-5955-42ed-8d2d-3981feabe59d.htseq.counts.gz_finished
239
bc26fe03-e143-4cdf-a1a0-c604f22d78e2.htseq.counts.gz_finished
240
bd595078-3d80-429a-8f18-d1f1981ba302.htseq.counts.gz_finished
241
30d69eb7-0afb-40dc-b243-c3ecf91a09fd.htseq.counts.gz_finished
242
1ec0cdf1-3bc2-4f86-bc5c-8d9cd30982d2.htseq.counts.gz_finished
243
f81798b2-8072-439f-b0a4-3268e5453188.htseq.counts.gz_finished
244
270677c6-3969-417e-b202-b1e49014445a.htseq.counts.gz_finished
245
f9415abb-ec64-4d0f-ad7e-26f2071b2c24.htseq.counts.gz_finished
246
749efeb5-db08-4ccd-8cd6-767ca80c85a8.htseq.counts.gz_finished
247
af995b92-6e87-480b-9d1e-4363ce7013a4.htseq.counts.gz_finished
248
bd3834fb-9a9b-4a7b-880f-b4073f51bd31.htseq.counts.gz_finished
249
43855b19-028a-4d3a-a825-98b17e9e51cf.htseq.counts.gz_finished
250
7868cab1-0300-493b-b910-6208a3c7921a.htseq.counts.gz_finished
251
74451263-2bde-4b6f-899f-d171915769b3.htseq.counts.gz_finished
252
dcbd5bcd-4a10-4e8b-bbe3-91682845c7db.htseq.counts.gz_finished
253
249869f5-8812-410c-89f8-2b7df632ac85.htseq.counts.gz_finished
254
b3ef98ef-0843-478d-a6d5-c42a1abccc53.htseq.counts.gz_finished
255
59e22196-c04d-46c9-b16f-98858dca67d1.htseq.counts.gz_finished
256
df398753-08fc-4a1d-90b1-a44760be2560.htseq.counts.gz_finished
257
54d174e1-fe50-4f92-b9cb-57160b705d83.htseq.counts.gz_finished
258
1d619122-94f2-4ced-81a3-475b6a510c3c.htseq.counts.gz_finished
259
cb76d1df-ba6e-479d-8858-005c17c94613.htseq.counts.gz_finished
260
ce2ffb35-774a-436b-87f9-3f86f2d4161e.htseq.counts.gz_finished
261
c61fb9d1-d808-4328-8a9c-7ccb4beb139f.htseq.counts.gz_finished
262
933f420d-7898-45c3-8b0f-bef9840aea7d.htseq.counts.gz_finished
263
0abe1897-c3a9-47e5-8a59-e5a2232b23db.htseq.counts.gz_finished
264
9e8b0d08-5be1-44c5-8dda-bd4d4ba3a88d.htseq.counts.gz_finished
265
ee6aaabb-4aa2-4578-bc14-27d311837005.htseq.counts.gz_finished
266
352a2653-8001-44cd-852c-288684114104.htseq.counts.gz_finished
267
b5bfd4bc-6740-4213-b31a-31d368a7319a.htseq.counts.gz_finished
268
e2e19d54-e271-4d94-af60-8a2fc470a22c.htseq.counts.gz_finished
269
b6cf2f29-6ea4-422e-bfa6-7bbc3e07e0b5.htseq.counts.gz_finished
270
394ef2d0-7bc0-444a-86d3-8f32385c2494.htseq.counts.gz_finished
271
90e6d83e-fb1b-44e9-8180-8c449c3de211.htseq.counts.gz_finished
272
643a2bf7-bfb7-4057-9e98-0c5fb479d052.htseq.counts.gz_finished
273
ef700f61-d8b0-43d0-92d1-feec949adc72.htseq.counts.gz_finished
274
7f0cf4dc-7b14-4852-bb70-110067ebc2a3.htseq.counts.gz_finished
275
0d2ee969-027b-4a7e-be4d-c44aec59c23b.htseq.counts.gz_finished
276
fced57a8-570a-4f05-aa10-4e16aa7ecc85.htseq.counts.gz_finished
277
fc822b3c-b73f-4529-960b-e3bdfcff80ed.htseq.counts.gz_finished
278
57511eed-4acc-4a58-8640-664e5710d003.htseq.counts.gz_finished
279
fc66f17a-2f6c-4dd9-8d9b-efd4baaca2af.htseq.counts.gz_finished
280
834f1044-cf26-4042-b7dd-50486170b28c.htseq.counts.gz_finished
281
aa3927eb-4e9f-4211-8e4f-a64bf8f5c53d.htseq.counts.gz_finished
282
4545348f-bbfe-4f68-826f-3fb2b019a328.htseq.counts.gz_finished
283
f3895ae4-1228-49b3-9342-3c3b86cb5243.htseq.counts.gz_finished
284
06c8dfda-b057-47cb-b435-9c2739694f6a.htseq.counts.gz_finished
285
0ee26933-679d-4e84-ab56-6b9581e5dbfd.htseq.counts.gz_finished
286
7844b74a-2846-4f42-8874-edcc90261fef.htseq.counts.gz_finished
287
e19da22a-2a4e-43ff-acff-12aeafa3a2c1.htseq.counts.gz_finished
288
b7b21fe3-2e21-4282-a707-da3c69a4453b.htseq.counts.gz_finished
289
74c84a22-7165-41f7-a38f-ed2db790ca2f.htseq.counts.gz_finished
290
f3e30992-70da-43fc-b0f4-69e53fa60474.htseq.counts.gz_finished
291
93e77514-c77b-4346-ae63-3e0d10512cff.htseq.counts.gz_finished
292
0776df15-e66e-4978-a585-0f0416fa6880.htseq.counts.gz_finished
293
a7202dc0-379c-40e6-9bbf-3337bd3328c8.htseq.counts.gz_finished
294
1ad09a17-bea8-40b7-81a0-9b5f94da574c.htseq.counts.gz_finished
295
e7616f66-c0a7-43f7-8295-f9d7a32f2b78.htseq.counts.gz_finished
296
b581fdd2-ede2-422b-817d-7c42d64a293c.htseq.counts.gz_finished
297
58dc0435-3e56-48c0-b76f-3fe1da6cf1dd.htseq.counts.gz_finished
298
d46b61b0-6f34-48ca-8c7d-23ae4d0b1a7c.htseq.counts.gz_finished
299
066a17b9-c724-4390-9a5f-d46cde556de3.htseq.counts.gz_finished
300
dc9b394f-3430-454e-960b-fc423c2eb0fb.htseq.counts.gz_finished
301
347eebe8-1e5e-484a-bab4-f84147eb9664.htseq.counts.gz_finished
302
3751b9c6-446b-4510-bb43-437ba4ffdc2e.htseq.counts.gz_finished
303
b06fbfec-fe8f-400a-b5be-664ac3958de4.htseq.counts.gz_finished
304
12531792-0d39-478f-ab05-90aceedf033d.htseq.counts.gz_finished
305
73bcbb60-38af-4143-95e9-1a842cdf9c20.htseq.counts.gz_finished
306
3e2a3ad6-cc31-4b8f-9738-3266a423ccd2.htseq.counts.gz_finished
307
31894369-d206-4948-8f75-fd7d05b0f578.htseq.counts.gz_finished
308
aee34477-7bfd-46c3-88a8-f3e7338a8444.htseq.counts.gz_finished
309
22c16f46-950f-4389-bb16-b5cec4d8b94e.htseq.counts.gz_finished
310
93084e1b-a6dd-4148-a251-2bb6bae40ddc.htseq.counts.gz_finished
311
f1d0dfc0-9d5f-415e-8a81-84e1c3b63609.htseq.counts.gz_finished
312
b2f50b85-591b-40af-b2a9-228a8523174b.htseq.counts.gz_finished
313
6d32c2f4-c318-4917-8e17-2eff74b5e127.htseq.counts.gz_finished
314
46b65e6b-ed6b-449c-b89e-c0b4f5afe7b6.htseq.counts.gz_finished
315
7b1bc957-02b8-46e8-84c3-f4cdf00b036a.htseq.counts.gz_finished
316
304a977a-e34e-4bf5-b4c0-3d06eb1d6519.htseq.counts.gz_finished
317
d90b08ef-512b-41b3-a329-45b0fd741587.htseq.counts.gz_finished
318
23a72fc4-9e50-4834-b5e0-863a5075c4a0.htseq.counts.gz_finished
319
54401c2a-7124-42b7-90d8-6267575bce51.htseq.counts.gz_finished
320
e1353694-9f60-442a-a164-5b2aca82419a.htseq.counts.gz_finished
321
6f74accd-5846-49f5-8a04-1b3926989071.htseq.counts.gz_finished
322
2d5ebcd0-4e47-4e76-adaf-21fa274ed0c4.htseq.counts.gz_finished
323
25b57a17-29ab-41f6-8809-45cf5b58a329.htseq.counts.gz_finished
324
376e643c-8ecb-49d2-95b0-e4f6571ea94e.htseq.counts.gz_finished
325
00511204-3512-4a5e-b664-60271e968903.htseq.counts.gz_finished
326
2b8bf629-3c22-4dcb-a9a5-ec01c5099167.htseq.counts.gz_finished
327
c6e85461-d986-46c3-b6b8-ab56d97320a3.htseq.counts.gz_finished
328
96833a09-14ce-4483-b17d-f4328e0b68f5.htseq.counts.gz_finished
329
bd0da177-9dad-4f37-9e6a-202b5fd2139e.htseq.counts.gz_finished
330
fd2f27db-566f-4742-b498-c6ac2b68fe3f.htseq.counts.gz_finished
331
4bf36a38-f5aa-4d9e-a1f4-15ab50ff1602.htseq.counts.gz_finished
332
f87e979d-2869-4742-84aa-30fd103a73fe.htseq.counts.gz_finished
333
f8e707b4-a897-4f17-a605-30d5be78ae93.htseq.counts.gz_finished
334
7223f19a-017e-465b-b81b-bed54252c6b0.htseq.counts.gz_finished
335
d770775c-efe5-4f6e-a211-1f8f5f184cd8.htseq.counts.gz_finished
336
5b1ae080-91c0-47ea-aa52-0bb72af1fb2f.htseq.counts.gz_finished
337
708413f6-ee3c-4379-a8a1-0143b0ac7adb.htseq.counts.gz_finished
338
596e208a-962b-45d9-b4f0-0eb244dd7874.htseq.counts.gz_finished
339
799199d9-0ff7-4d58-97bd-08009f64947d.htseq.counts.gz_finished
340
731af067-9958-4967-b638-fc2a8a6bcf1f.htseq.counts.gz_finished
341
98f08602-3393-436d-81d4-447ec38444b5.htseq.counts.gz_finished
342
23f8f5e2-25b8-4b43-b939-66e7e3f55977.htseq.counts.gz_finished
343
7424bd55-ba00-4aea-b4d6-b827b4f60902.htseq.counts.gz_finished
344
e2730969-cb85-467f-9ba2-8a9d521ec06a.htseq.counts.gz_finished
345
b58a2839-3089-4164-b66c-9b51873f25f8.htseq.counts.gz_finished
346
8f769163-f20b-4331-a415-3b49813c6a3a.htseq.counts.gz_finished
347
99aa28e6-25ae-4137-adbd-339cd07b1bba.htseq.counts.gz_finished
348
8c7a9e45-89aa-4a18-addb-dd1887ec7e15.htseq.counts.gz_finished
349
acf02cd5-270d-40c0-864d-abebe048f309.htseq.counts.gz_finished
350
df161225-6334-45af-a74a-a71af0f7b2ae.htseq.counts.gz_finished
351
4cb92a66-9f3c-4b86-a6e0-0244d26e1985.htseq.counts.gz_finished
352
510293b0-9812-443f-9ffe-3c3ce78528f6.htseq.counts.gz_finished
353
76c7ff87-5595-47d6-8c41-69de3d8afb10.htseq.counts.gz_finished
354
6d7f5731-e599-4e3f-b80b-1f0ac2f68111.htseq.counts.gz_finished
355
ccdbc757-4bf2-4132-bc03-dced7d6c2c71.htseq.counts.gz_finished
356
437ad095-c2bd-475f-ba1d-a5ffdd17ab7f.htseq.counts.gz_finished
357
082418be-42c4-4717-95f4-755ed21d4366.htseq.counts.gz_finished
358
16009a5d-2149-4fa5-8e66-faac0f9f0309.htseq.counts.gz_finished
359
3eb03db7-efc1-4f20-bbc9-61551905eb9b.htseq.counts.gz_finished
360
71c3f9c6-9226-4a8f-910a-3cee62dbeb3c.htseq.counts.gz_finished
361
5d5c9e4d-23d8-4e99-9722-fe5d5e35941d.htseq.counts.gz_finished
362
8e825388-620f-4d87-8ed3-84ae6b5b7dbe.htseq.counts.gz_finished
363
587e44e4-87ba-4981-a520-d20612486f53.htseq.counts.gz_finished
364
c7bd5f7a-6511-4d4c-b8b7-5bd3cc64036b.htseq.counts.gz_finished
365
338b949b-f16c-4569-84c2-e2c2d60f2a82.htseq.counts.gz_finished
366
18c32c46-00f9-4437-bdbb-05233fdd676e.htseq.counts.gz_finished
367
6017c75f-8410-410b-91fc-54035008c1d9.htseq.counts.gz_finished
368
631a4c93-c41d-4ca3-84c7-6f6ad226397a.htseq.counts.gz_finished
369
b393f2a6-56ad-4467-9051-98c0cfa0550c.htseq.counts.gz_finished
370
98cf4f9b-aa89-4c14-b84b-2b8f32439ea0.htseq.counts.gz_finished
371
dbb50b30-9856-4a56-91e1-d31ad106fd48.htseq.counts.gz_finished
372
de0262c2-bccb-418d-acc3-4ce4afe3e78e.htseq.counts.gz_finished
373
a9d1ff39-da59-4c95-a402-1793e6c9f5e4.htseq.counts.gz_finished
374
6a04b8ef-00ac-45d7-ac72-2e311e383f35.htseq.counts.gz_finished
375
175e2233-28a0-47c7-b395-19dd033b79d8.htseq.counts.gz_finished
376
b0ae2bb5-9b1b-47a3-abc7-76c02bf8a229.htseq.counts.gz_finished
377
38f141c2-4d24-45a7-bc7f-ecdba7082f72.htseq.counts.gz_finished
378
dc724482-dc82-41e0-9f01-3451e5be2ae8.htseq.counts.gz_finished
379
dd1d19b2-681b-447b-b16f-f9b0bf4f4c30.htseq.counts.gz_finished
380
b16f9fe4-d8fe-4f81-9422-20447c909f0a.htseq.counts.gz_finished
381
abe20df7-6b97-4397-8864-881bac27e92c.htseq.counts.gz_finished
382
a1f37975-6bc2-4583-9035-8e8c70677833.htseq.counts.gz_finished
383
3d3245f2-a2c9-4742-bfec-cd17f5aaca23.htseq.counts.gz_finished
384
463f2d23-8b4d-45f5-9600-06c2920498ca.htseq.counts.gz_finished
385
23f04c87-a093-4a1a-a1a5-d2928cc3c78b.htseq.counts.gz_finished
386
94e2fc89-29f8-4a60-8d42-d43e01530839.htseq.counts.gz_finished
387
d7047346-49fd-444e-84d2-c3e4b7cda15b.htseq.counts.gz_finished
388
d8f60cfe-3faa-406f-8ed6-2bb201dc36f4.htseq.counts.gz_finished
389
6a73a6c5-ec81-4179-9270-47b4a99aabe5.htseq.counts.gz_finished
390
f2b8d302-9b7a-4055-835b-fee9d53ea9e0.htseq.counts.gz_finished
391
d40bae5d-74ea-4f68-890b-38ccc04e3e33.htseq.counts.gz_finished
392
bd9949b8-2b83-412c-be3f-64d2af581660.htseq.counts.gz_finished
393
3d8a6f49-654a-4f69-93d0-5c7f4f2159f6.htseq.counts.gz_finished
394
2e1b2787-8e16-4622-a1e0-3b96e6381e87.htseq.counts.gz_finished
395
7afccfe4-4245-4f6c-9a56-b1ded4af45a3.htseq.counts.gz_finished
396
052d9c90-7474-485b-b781-455cde6a717d.htseq.counts.gz_finished
397
e4e7ec56-30aa-48d4-aef4-7e88f408a2f8.htseq.counts.gz_finished
398
d1245d77-44fd-4238-82ce-b52bc519a24a.htseq.counts.gz_finished
399
ee79e954-2c9a-44ee-95be-03763ea5c395.htseq.counts.gz_finished
400
5c50c128-f5df-4000-9e2b-3c261c02535f.htseq.counts.gz_finished
401
22a09102-9a4f-4587-9014-24283c663ca6.htseq.counts.gz_finished
402
8039b179-b990-4963-b565-4a90afe89b0f.htseq.counts.gz_finished
403
562fdef0-1529-45b3-aef6-79ffab91a0e6.htseq.counts.gz_finished
404
ecefab08-36f5-4a49-9e4d-9e6def0d747e.htseq.counts.gz_finished
405
41f067e1-7d50-448c-86fd-bdad3824b2ec.htseq.counts.gz_finished
406
786537c7-34b1-4d87-bbe4-f28b2df869b2.htseq.counts.gz_finished
407
22562d81-b588-454e-bf1a-c67bd24f0d54.htseq.counts.gz_finished
408
33b759c0-0b57-403a-8426-122df5881832.htseq.counts.gz_finished
409
a82ab02e-4894-43f3-adf3-8a9faebee4e3.htseq.counts.gz_finished
410
5532e9a2-66b8-4dd2-aa1a-ebbb08163280.htseq.counts.gz_finished
411
f4b179c1-9d27-439b-a1ef-3dfbbd684c15.htseq.counts.gz_finished
412
961f2c3c-1914-4b25-99ad-2eafaad4ab43.htseq.counts.gz_finished
413
23b10ac4-abaa-44e3-81b1-e2e7f7149e3c.htseq.counts.gz_finished
414
111d998f-4133-436a-9990-64cb293a986e.htseq.counts.gz_finished
415
661f20c6-6d4f-4a6a-b819-a3d1543bad27.htseq.counts.gz_finished
416
af504400-7f49-4561-a8e4-8b0b6ca3d02d.htseq.counts.gz_finished
417
2f05c173-adfa-40c3-baf8-02563270ff3e.htseq.counts.gz_finished
418
7c169d29-58cd-4f77-9e3e-eae41234e936.htseq.counts.gz_finished
419
1f920941-0118-4f2f-87c2-f3cede2d5f0b.htseq.counts.gz_finished
420
merged[1]_____no_output_____#TPM 是 (reads/exonlength)*10^6 / sum(reads/exonlength)_____no_output_____# test = files[27]
# print(test)
# testpath = path + test
# fileName = os.listdir(testpath)_____no_output_____# def getfilename(filepath):
#     fileName = os.listdir(filepath)
#     maxlength = 0
#     maxindex = 0
#     for index in range(0,len(fileName)):
#         thisname = fileName[i]
#         if maxlength <= len(thisname):
#             maxlength = len(thisname)
#             maxindex = index
#
#     return fileName[maxindex]
        _____no_output_____
</code>


# Notebook from LDWLab/DESIRE
Path: populate_db/Notebooks/Design (Petar Penev's conflicted copy 2019-06-14).ipynb

# Timeline for scripts of the SEREB database

![Image_001.png](attachment:Image_001.png)

Each outline describes a script for updating the database. **Black** outlines are scripts that do not require prior tables to be completed.  Cyan outline is for scripts that require tables from black scripts.  Yellow  requires tables from Cyan and bellow._____no_output_____
## Surrogate keys
As of now the major question is whether we want auto-incrementing (AI) keys (aka *surrogate keys*) for most of the tables. This will make things easier in the future, but markedly harder now. The scripts which update the tables will need to query tables for the surrogate keys and associate them with the correct data in order to update other tables.

We will HAVE to use surrogate keys for tables like **Gene_sequences_type**. (Substituted for Gene_metadata)

### Nomenclature
I would like to try as a test and see how surrogate keys would work in the **Nomenclature** and **Old_name** tables. With *nom_id* being an AI surrogate key for **Nomenclature** and *nomo_id* being a foreign key of *nom_id* in the **Old_name** table.

The idea of the current design for **Nomenclature** and **Old_name** is to allow for multiple repeated *new_name* entries, which have different *occurence*s (which can specify both the domain and the cellular localization - B,A,E (assumes cytozol) or m,c). The the association with **Old_name** is done through the unique AI surrogate key *nom_id*. *old_id* will also be AI.

Because there were different names used for *H. sapiens* and *S. cerevisiae* we might need another column in **Old_name** which specifies that (the *occurence* will be the same since they are both E). **DONE** (*yeast-human*)

### Species
For some model organisms there are many strains/substrains etc. For now I have experinced this with *E. coli K-12* (txid 83333). In my work up to now this was supposed to be the leaf identifier for the *E. coli* specie, however it seems that there are many substrains which show up when we query Entrez for all the genes from txid 83333:

> esearch -db gene -query "txid83333[Organism]" | efetch -format tabular

This returns many results from different substrains of the K-12. I propose we use **511145** *Escherichia coli str. K-12 substr. MG1655* as our leaf identifier. In principle this is not a cause to change the design of the **Species**, **Species_TaxGroup**, and **TaxGroups** tables. All that has to be done is check for the model organisms that would show similar behaviour (for example *S. cerevisiae*) and add the appropriate leaf taxids in the **Species** table, as well as the connections to the **Species_TaxGroup**. This should already be possible through Vasanta's script.

### Gene_metadata (previously Gene_Sequences_Type)

#### Reverse complement nucleotidic sequences
We will have for starters 3 *polymer_type*s (in the future we can add stuff like depsipeptides) in the **Gene_Sequences_Type**. One *protein*, one back-translated *nucl*eotidic (from the transl-aln.py script) and one *nucl_rev* which will be the reverse complement of the *nucl*.

### Residues
In essence should be the heart of the database, since it distinguishes our data at the lowest level.
Should define positions of residues in the protein/RNA raw sequence (**Gene_metadata**.*Fullseq*). It will:

1. Send foreign keys to **Alignment_Data** defining which residue is in what position (*alignment_pos*) based on a particular alignment.
2. Send foreign keys to **Secondary_structure_residues** defining which residue goes in which secondary structure (with key *variation*, *molecule_group*, and *specie*) and it's *map_index* (= *res2index*) (position in relation to other participants in the SSE).
3. Send foreign keys to **Tertiary_structure_residues** defining which residue goes in which 3D structure (definied by *chain* and *pdb_id* (should be inherently connected to *specie*)) The difference in numbering will be resolved in here (*resAltNum* = *res3index*)_____no_output_____# Merging Sequence and Structure DBs

Our aim is to achieve complete integration between sequence and structure data. There are 4 pairs of tables which will act as interface between the two databases:

**Sequence** - **Structure**
1. **Species** - **Species**
2. **Residues** - **Residues**
3. **Alignment** - **Secondary_structure_residues** (Likely not needed)
4. **Gene_data** - **Polymers**

### Species  PROGRESSING
Does not require much reworking, we should just add column *Abbreviation* and merge the two tables.

### Residues PROGRESSING
Should move *map_index* from the structural residues to **Secondary_structure_residues**.

Should connect through **SS_Data** in a m-n relationship with **SecondaryStructures**.

### Alignment REMOVED
Not yet conceptualized. (Not needed)

### Polymer_Data DONE previously Gene_data
Many columns in **Polymers** are redundant with information present in other tables:
- *molecule_name_long* - present in **Nomenclature**.*new_name* or *gene_symbol* and *gene_description* in **Gene_Data**.
- *molecule_name_short* - unneded (can be treated as another old name in **Old_name**)
- *Molecule_type* - same as **Gene_metadata**.*polymer_type*
- *Molecule_group* - for rProteins is already present in their name (L or S), however it might be needed for rRNA and other proteins. It should be added to **Gene_metadata**. Can be in Nomenclature?

_____no_output_____# Redesign due to discontinued problems with NCBI Gene DB

Instead of **Gene_sequences_type** we have a metadata table for the genes. We will not use the gene identifier so strictly, since several species are missing entries in the Gene DB. Furthermore, for many entries in the Gene DB a discontinued label has been applied. To solve this we will use GI when available and when it is not we will use protein accession. The metadata table will hold information which accession is being used and what is the polymer, sine the Gene_Data PK will be an auto-incremental surrogate key, this shouldn't pose any problems.

The other tables connecting to **Gene_sequences_type** will connect to **Gene_Data** instead._____no_output_____# Residues will connect sequence and structural data

_____no_output_____


# Notebook from Tensorflow-Devs/probability
Path: tensorflow_probability/examples/jupyter_notebooks/Undocumented_Infection_and_the_Dissemination_of_SARS-CoV2.ipynb

##### Copyright 2020 The TensorFlow Authors.
##### Copyright 2020 Sen Pei (Columbia University).

Licensed under the Apache License, Version 2.0 (the "License");_____no_output_____

#@title Licensed under the Apache License, Version 2.0 (the "License"); { display-mode: "form" }
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License._____no_output_____

# Substantial Undocumented Infection Facilitates the Rapid Dissemination of Novel Coronavirus (SARS-CoV2)

    View on TensorFlow.org

    Run in Google Colab

    View source on GitHub

    Download notebook

_____no_output_____This is a TensorFlow Probability port of the eponymous 16 March 2020 paper by Li et al.  We faithfully reproduce the original authors' methods and results on the TensorFlow Probability platform, showcasing some of TFP's capabilities in the setting of modern epidemiology modeling.  Porting to TensorFlow gives us a ~10x speedup relative to the original Matlab code, and, since TensorFlow Probability pervasively supports vectorized batch computation, also favorably scales to hundreds of independent replications.

### Original paper

Ruiyun Li, Sen Pei, Bin Chen, Yimeng Song, Tao Zhang, Wan Yang, and Jeffrey Shaman. Substantial undocumented infection facilitates the rapid dissemination of novel coronavirus (SARS-CoV2). (2020), doi:
https://doi.org/10.1126/science.abb3221 .

*Abstract:* "Estimation of the prevalence and contagiousness of undocumented novel coronavirus (SARS-CoV2) infections is critical for understanding the overall prevalence and pandemic potential of this disease. Here we use observations of reported infection within China, in conjunction with mobility data, a networked dynamic metapopulation model and Bayesian inference, to infer critical epidemiological characteristics associated with SARS-CoV2, including the fraction of undocumented infections and their contagiousness. We estimate 86% of all infections were undocumented (95% CI: [82%–90%]) prior to 23 January 2020 travel restrictions. Per person, the transmission rate of undocumented infections was 55% of documented infections ([46%–62%]), yet, due to their greater numbers, undocumented infections were the infection source for 79% of documented cases. These findings explain the rapid geographic spread of SARS-CoV2 and indicate containment of this virus will be particularly challenging."

[Github link](https://github.com/SenPei-CU/COVID-19) to the code and data._____no_output_____## Overview_____no_output_____The model is a [compartmental disease model](https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology), with compartments for "susceptible", "exposed" (infected but not yet infectious), "never-documented infectious", and "eventually-documented infectious". There are two noteworthy features: separate compartments for each of 375 Chinese cities, with an assumption about how people travel from one city to another; and delays in reporting infection, so that a case that becomes "eventually-documented infectious" on day $t$ doesn't show up in the observed case counts until a stochastic later day.

The model assumes that the never-documented cases end up undocumented by being milder, and thus infect others at a lower rate.  The main parameter of interest in the original paper is the proportion of cases that go undocumented, to estimate both the extent of existing infection, and the impact of undocumented transmission on the spread of the disease.

This colab is structured as a code walkthrough in bottom-up style.  In order, we will
- Ingest and briefly examine the data,
- Define the state space and dynamics of the model,
- Build up a suite of functions for doing inference in the model following Li et al, and
- Invoke them and examine the results.  Spoiler: They come out the same as the paper.
_____no_output_____## Installation and Python Imports_____no_output_____

!pip3 install -q tf-nightly tfp-nightly_____no_output_____from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import io
import requests
import time
import zipfile

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

import tensorflow.compat.v2 as tf
import tensorflow_probability as tfp
from tensorflow_probability.python.internal import samplers

tfd = tfp.distributions
tfes = tfp.experimental.sequential_____no_output_____

## Data Import

Let's import the data from github and inspect some of it._____no_output_____

r = requests.get('https://raw.githubusercontent.com/SenPei-CU/COVID-19/master/Data.zip')
z = zipfile.ZipFile(io.BytesIO(r.content))
z.extractall('/tmp/')
raw_incidence = pd.read_csv('/tmp/data/Incidence.csv')
raw_mobility = pd.read_csv('/tmp/data/Mobility.csv')
raw_population = pd.read_csv('/tmp/data/pop.csv')_____no_output_____

Below we can see the raw incidence count per day. We are most interested in the first 14 days (January 10th to January 23rd), as the travel restrictions were put in place on the 23rd.  The paper deals with this by modeling Jan 10-23 and Jan 23+ separately, with different parameters; we will just restrict our reproduction to the earlier period._____no_output_____

raw_incidence.drop('Date', axis=1)  # The 'Date' column is all 1/18/21
# Luckily the days are in order, starting on January 10th, 2020._____no_output_____

Let's sanity-check the Wuhan incidence counts._____no_output_____

plt.plot(raw_incidence.Wuhan, '.-')
plt.title('Wuhan incidence counts over 1/10/20 - 02/08/20')
plt.show()_____no_output_____

So far, so good.  Now the initial population counts._____no_output_____

raw_population_____no_output_____

Let's also check and record which entry is Wuhan._____no_output_____

raw_population['City'][169]_____no_output_____WUHAN_IDX = 169_____no_output_____

And here we see the mobility matrix between different cities. This is a proxy for the number of people moving between different cities on the first 14 days.  It's dervied from GPS records provided by Tencent for the 2018 Lunar New Year season.  Li et al model mobility during the 2020 season as some unknown (subject to inference) constant factor $\theta$ times this._____no_output_____

raw_mobility_____no_output_____

Finally, let's preprocess all this into numpy arrays that we can consume._____no_output_____

# The given populations are only "initial" because of intercity mobility during
# the holiday season.
initial_population = raw_population['Population'].to_numpy().astype(np.float32)_____no_output_____

Convert the mobility data into an [L, L, T]-shaped Tensor, where L is the number of locations, and T is the number of timesteps._____no_output_____

daily_mobility_matrices = []
for i in range(1, 15):
  day_mobility = raw_mobility[raw_mobility['Day'] == i]

  # Make a matrix of daily mobilities.
  z = pd.crosstab(
      day_mobility.Origin,
      day_mobility.Destination,
      values=day_mobility['Mobility Index'], aggfunc='sum', dropna=False)

  # Include every city, even if there are no rows for some in the raw data on
  # some day.  This uses the sort order of `raw_population`.
  z = z.reindex(index=raw_population['City'], columns=raw_population['City'],
                fill_value=0)
  # Finally, fill any missing entries with 0. This means no mobility.
  z = z.fillna(0)
  daily_mobility_matrices.append(z.to_numpy())

mobility_matrix_over_time = np.stack(daily_mobility_matrices, axis=-1).astype(
    np.float32)_____no_output_____

Finally take the observed infections and make an [L, T] table._____no_output_____

# Remove the date parameter and take the first 14 days.
observed_daily_infectious_count = raw_incidence.to_numpy()[:14, 1:]
observed_daily_infectious_count = np.transpose(
    observed_daily_infectious_count).astype(np.float32)_____no_output_____

And double-check that we got the shapes the way we wanted.  As a reminder, we're working with 375 cities and 14 days._____no_output_____

print('Mobility Matrix over time should have shape (375, 375, 14): {}'.format(
    mobility_matrix_over_time.shape))
print('Observed Infectious should have shape (375, 14): {}'.format(
    observed_daily_infectious_count.shape))
print('Initial population should have shape (375): {}'.format(
    initial_population.shape))Mobility Matrix over time should have shape (375, 375, 14): (375, 375, 14)
Observed Infectious should have shape (375, 14): (375, 14)
Initial population should have shape (375): (375,)

## Defining State and Parameters

Let's start defining our model. The model we are reproducing is a variant of an [SEIR model](https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology#The_SEIR_model). In this case we have the following time-varying states:
* $S$: Number of people susceptible to the disease in each city.
* $E$: Number of people in each city exposed to the disease but not infectious yet.  Biologically, this corresponds to contracting the disease, in that all exposed people eventually become infectious.
* $I^u$: Number of people in each city who are infectious but undocumented.  In the model, this actually means "will never be documented".
* $I^r$: Number of people in each city who are infectious and documented as such.  Li et al model reporting delays, so $I^r$ actually corresponds to something like "case is severe enough to be documented at some point in the future".

As we will see below, we will be inferring these states by running an Ensemble-adjusted Kalman Filter (EAKF) forward in time.  The state vector of the EAKF is one city-indexed vector for each of these quantities.

The model has the following inferrable global, time-invariant parameters:

* $\beta$: The transmission rate due to documented-infectious individuals.
* $\mu$: The relative transmission rate due to undocumented-infectious
  individuals. This will act through the product $\mu \beta$.
* $\theta$: The intercity mobility factor. This is a factor greater than
  1 correcting for underreporting of mobility data (and for population growth
  from 2018 to 2020).
* $Z$: The average incubation period (i.e., time in the "exposed" state).
* $\alpha$: This is the fraction of infections severe enough to be (eventually) documented.
* $D$: The average duration of infections (i.e., time in either "infectious" state).

We will be inferring point estimates for these parameters with an Iterative-Filtering loop around the EAKF for the states.

The model also depends on un-inferred constants:
* $M$: The intercity mobility matrix.  This is time-varying and presumed given.  Recall that it's scaled by the inferred parameter $\theta$ to give the actual population movements between cities.
* $N$: The total number of people in each city.  The initial populations are taken as given, and the time-variation of population is computed from the mobility numbers $\theta M$.
_____no_output_____First, we give ourselves some data structures for holding our states and parameters.
_____no_output_____

SEIRComponents = collections.namedtuple(
  typename='SEIRComponents',
  field_names=[
    'susceptible',              # S
    'exposed',                  # E
    'documented_infectious',    # I^r
    'undocumented_infectious',  # I^u
    # This is the count of new cases in the "documented infectious" compartment.
    # We need this because we will introduce a reporting delay, between a person
    # entering I^r and showing up in the observable case count data.
    # This can't be computed from the cumulative `documented_infectious` count,
    # because some portion of that population will move to the 'recovered'
    # state, which we aren't tracking explicitly.
    'daily_new_documented_infectious'])

ModelParams = collections.namedtuple(
    typename='ModelParams',
    field_names=[
      'documented_infectious_tx_rate',             # Beta
      'undocumented_infectious_tx_relative_rate',  # Mu
      'intercity_underreporting_factor',           # Theta
      'average_latency_period',                    # Z
      'fraction_of_documented_infections',         # Alpha
      'average_infection_duration'                 # D
    ]
)_____no_output_____

We also code Li et al's bounds for the values of the parameters._____no_output_____

PARAMETER_LOWER_BOUNDS = ModelParams(
    documented_infectious_tx_rate=0.8,
    undocumented_infectious_tx_relative_rate=0.2,
    intercity_underreporting_factor=1.,
    average_latency_period=2.,
    fraction_of_documented_infections=0.02,
    average_infection_duration=2.
)

PARAMETER_UPPER_BOUNDS = ModelParams(
    documented_infectious_tx_rate=1.5,
    undocumented_infectious_tx_relative_rate=1.,
    intercity_underreporting_factor=1.75,
    average_latency_period=5.,
    fraction_of_documented_infections=1.,
    average_infection_duration=5.
)_____no_output_____

## SEIR Dynamics

Here we define the relationship between the parameters and state.

The time-dynamics equations from Li et al (supplemental material, eqns 1-5) are as follows:

$\frac{dS_i}{dt} = -\beta \frac{S_i I_i^r}{N_i} - \mu \beta \frac{S_i I_i^u}{N_i} + \theta \sum_k \frac{M_{ij} S_j}{N_j - I_j^r} - + \theta \sum_k \frac{M_{ji} S_j}{N_i - I_i^r}$

$\frac{dE_i}{dt} = \beta \frac{S_i I_i^r}{N_i} + \mu \beta \frac{S_i I_i^u}{N_i} -\frac{E_i}{Z} + \theta \sum_k \frac{M_{ij} E_j}{N_j - I_j^r} - + \theta \sum_k \frac{M_{ji} E_j}{N_i - I_i^r}$

$\frac{dI^r_i}{dt} = \alpha \frac{E_i}{Z} - \frac{I_i^r}{D}$

$\frac{dI^u_i}{dt} = (1 - \alpha) \frac{E_i}{Z} - \frac{I_i^u}{D} + \theta \sum_k \frac{M_{ij} I_j^u}{N_j - I_j^r} - + \theta \sum_k \frac{M_{ji} I^u_j}{N_i - I_i^r}$

$N_i = N_i + \theta \sum_j M_{ij} - \theta \sum_j M_{ji}$

As a reminder, the $i$ and $j$ subscripts index cities.  These equations model the time-evolution of the disease through
- Contact with infectious individuals leading to more infection;
- Disease progression from "exposed" to one of the "infectious" states;
- Disease progression from "infectious" states to recovery, which we model by removal from the modeled population;
- Inter-city mobility, including exposed or undocumented-infectious persons; and
- Time-variation of daily city populations through inter-city mobility.

Following Li et al, we assume that people with cases severe enough to eventually be reported do not travel between cities.

Also following Li et al, we treat these dynamics as subject to term-wise Poisson noise, i.e., each term is actually the rate of a Poisson, a sample from which gives the true change.  The Poisson noise is term-wise because subtracting (as opposed to adding) Poisson samples does not yield a Poisson-distributed result.

We will evolve these dynamics forward in time with the classic fourth-order Runge-Kutta integrator, but first let's define the function that computes them (including sampling the Poisson noise)._____no_output_____

def sample_state_deltas(
    state, population, mobility_matrix, params, seed, is_deterministic=False):
  """Computes one-step change in state, including Poisson sampling.

  Note that this is coded to support vectorized evaluation on arbitrary-shape
  batches of states.  This is useful, for example, for running multiple
  independent replicas of this model to compute credible intervals for the
  parameters.  We refer to the arbitrary batch shape with the conventional
  `B` in the parameter documentation below.  This function also, of course,
  supports broadcasting over the batch shape.

  Args:
    state: A `SEIRComponents` tuple with fields Tensors of shape
      B + [num_locations] giving the current disease state.
    population: A Tensor of shape B + [num_locations] giving the current city
      populations.
    mobility_matrix: A Tensor of shape B + [num_locations, num_locations] giving
      the current baseline inter-city mobility.
    params: A `ModelParams` tuple with fields Tensors of shape B giving the
      global parameters for the current EAKF run.
    seed: Initial entropy for pseudo-random number generation.  The Poisson
      sampling is repeatable by supplying the same seed.
    is_deterministic: A `bool` flag to turn off Poisson sampling if desired.

  Returns:
    delta: A `SEIRComponents` tuple with fields Tensors of shape
      B + [num_locations] giving the one-day changes in the state, according
      to equations 1-4 above (including Poisson noise per Li et al).
  """
  undocumented_infectious_fraction = state.undocumented_infectious / population
  documented_infectious_fraction = state.documented_infectious / population

  # Anyone not documented as infectious is considered mobile
  mobile_population = (population - state.documented_infectious)
  def compute_outflow(compartment_population):
    raw_mobility = tf.linalg.matvec(
        mobility_matrix, compartment_population / mobile_population)
    return params.intercity_underreporting_factor * raw_mobility
  def compute_inflow(compartment_population):
    raw_mobility = tf.linalg.matmul(
        mobility_matrix,
        (compartment_population / mobile_population)[..., tf.newaxis],
        transpose_a=True)
    return params.intercity_underreporting_factor * tf.squeeze(
        raw_mobility, axis=-1)

  # Helper for sampling the Poisson-variate terms.
  seeds = samplers.split_seed(seed, n=11)
  if is_deterministic:
    def sample_poisson(rate):
      return rate
  else:
    def sample_poisson(rate):
      return tfd.Poisson(rate=rate).sample(seed=seeds.pop())

  # Below are the various terms called U1-U12 in the paper. We combined the
  # first two, which should be fine; both are poisson so their sum is too, and
  # there's no risk (as there could be in other terms) of going negative.
  susceptible_becoming_exposed = sample_poisson(
      state.susceptible *
      (params.documented_infectious_tx_rate *
       documented_infectious_fraction +
       (params.undocumented_infectious_tx_relative_rate *
        params.documented_infectious_tx_rate) *
       undocumented_infectious_fraction))  # U1 + U2

  susceptible_population_inflow = sample_poisson(
      compute_inflow(state.susceptible))  # U3
  susceptible_population_outflow = sample_poisson(
      compute_outflow(state.susceptible))  # U4

  exposed_becoming_documented_infectious = sample_poisson(
      params.fraction_of_documented_infections *
      state.exposed / params.average_latency_period)  # U5
  exposed_becoming_undocumented_infectious = sample_poisson(
      (1 - params.fraction_of_documented_infections) *
      state.exposed / params.average_latency_period)  # U6

  exposed_population_inflow = sample_poisson(
      compute_inflow(state.exposed))  # U7
  exposed_population_outflow = sample_poisson(
      compute_outflow(state.exposed))  # U8

  documented_infectious_becoming_recovered = sample_poisson(
      state.documented_infectious /
      params.average_infection_duration)  # U9
  undocumented_infectious_becoming_recovered = sample_poisson(
      state.undocumented_infectious /
      params.average_infection_duration)  # U10

  undocumented_infectious_population_inflow = sample_poisson(
      compute_inflow(state.undocumented_infectious))  # U11
  undocumented_infectious_population_outflow = sample_poisson(
      compute_outflow(state.undocumented_infectious))  # U12

  # The final state_deltas
  return SEIRComponents(
      # Equation [1]
      susceptible=(-susceptible_becoming_exposed +
                   susceptible_population_inflow +
                   -susceptible_population_outflow),
      # Equation [2]
      exposed=(susceptible_becoming_exposed +
               -exposed_becoming_documented_infectious +
               -exposed_becoming_undocumented_infectious +
               exposed_population_inflow +
               -exposed_population_outflow),
      # Equation [3]
      documented_infectious=(
          exposed_becoming_documented_infectious +
          -documented_infectious_becoming_recovered),
      # Equation [4]
      undocumented_infectious=(
          exposed_becoming_undocumented_infectious +
          -undocumented_infectious_becoming_recovered +
          undocumented_infectious_population_inflow +
          -undocumented_infectious_population_outflow),
      # New to-be-documented infectious cases, subject to the delayed
      # observation model.
      daily_new_documented_infectious=exposed_becoming_documented_infectious)_____no_output_____

Here's the integrator.  This is completely standard, except for passing the PRNG seed through to the `sample_state_deltas` function to get independent Poisson noise at each of the partial steps that the Runge-Kutta method calls for._____no_output_____

@tf.function(autograph=False)
def rk4_one_step(state, population, mobility_matrix, params, seed):
  """Implement one step of RK4, wrapped around a call to sample_state_deltas."""
  # One seed for each RK sub-step
  seeds = samplers.split_seed(seed, n=4)

  deltas = tf.nest.map_structure(tf.zeros_like, state)
  combined_deltas = tf.nest.map_structure(tf.zeros_like, state)

  for a, b in zip([1., 2, 2, 1.], [6., 3., 3., 6.]):
    next_input = tf.nest.map_structure(
        lambda x, delta, a=a: x + delta / a, state, deltas)
    deltas = sample_state_deltas(
        next_input,
        population,
        mobility_matrix,
        params,
        seed=seeds.pop(), is_deterministic=False)
    combined_deltas = tf.nest.map_structure(
        lambda x, delta, b=b: x + delta / b, combined_deltas, deltas)

  return tf.nest.map_structure(
      lambda s, delta: s + tf.round(delta),
      state, combined_deltas)_____no_output_____

## Initialization

Here we implement the initialization scheme from the paper.

Following Li et al, our inference scheme will be an ensemble adjustment Kalman filter inner loop, surrounded by an iterated filtering outer loop (IF-EAKF).  Computationally, that means we need three kinds of initialization:
- Initial state for the inner EAKF
- Initial parameters for the outer IF, which are also the initial
  parameters for the first EAKF
- Updating parameters from one IF iteration to the next, which serve
  as the initial parameters for each EAKF other than the first.

_____no_output_____

def initialize_state(num_particles, num_batches, seed):
  """Initialize the state for a batch of EAKF runs.

  Args:
    num_particles: `int` giving the number of particles for the EAKF.
    num_batches: `int` giving the number of independent EAKF runs to
      initialize in a vectorized batch.
    seed: PRNG entropy.

  Returns:
    state: A `SEIRComponents` tuple with Tensors of shape [num_particles,
      num_batches, num_cities] giving the initial conditions in each
      city, in each filter particle, in each batch member.
  """
  num_cities = mobility_matrix_over_time.shape[-2]
  state_shape = [num_particles, num_batches, num_cities]
  susceptible = initial_population * np.ones(state_shape, dtype=np.float32)
  documented_infectious = np.zeros(state_shape, dtype=np.float32)
  daily_new_documented_infectious = np.zeros(state_shape, dtype=np.float32)

  # Following Li et al, initialize Wuhan with up to 2000 people exposed
  # and another up to 2000 undocumented infectious.
  rng = np.random.RandomState(seed[0] % (2**31 - 1))
  wuhan_exposed = rng.randint(
      0, 2001, [num_particles, num_batches]).astype(np.float32)
  wuhan_undocumented_infectious = rng.randint(
      0, 2001, [num_particles, num_batches]).astype(np.float32)

  # Also following Li et al, initialize cities adjacent to Wuhan with three
  # days' worth of additional exposed and undocumented-infectious cases,
  # as they may have traveled there before the beginning of the modeling
  # period.
  exposed = 3 * mobility_matrix_over_time[
      WUHAN_IDX, :, 0] * wuhan_exposed[
          ..., np.newaxis] / initial_population[WUHAN_IDX]
  undocumented_infectious = 3 * mobility_matrix_over_time[
      WUHAN_IDX, :, 0] * wuhan_undocumented_infectious[
          ..., np.newaxis] / initial_population[WUHAN_IDX]

  exposed[..., WUHAN_IDX] = wuhan_exposed
  undocumented_infectious[..., WUHAN_IDX] = wuhan_undocumented_infectious

  # Following Li et al, we do not remove the inital exposed and infectious
  # persons from the susceptible population.
  return SEIRComponents(
      susceptible=tf.constant(susceptible),
      exposed=tf.constant(exposed),
      documented_infectious=tf.constant(documented_infectious),
      undocumented_infectious=tf.constant(undocumented_infectious),
      daily_new_documented_infectious=tf.constant(daily_new_documented_infectious))

def initialize_params(num_particles, num_batches, seed):
  """Initialize the global parameters for the entire inference run.

  Args:
    num_particles: `int` giving the number of particles for the EAKF.
    num_batches: `int` giving the number of independent EAKF runs to
      initialize in a vectorized batch.
    seed: PRNG entropy.

  Returns:
    params: A `ModelParams` tuple with fields Tensors of shape
      [num_particles, num_batches] giving the global parameters
      to use for the first batch of EAKF runs.
  """
  # We have 6 parameters. We'll initialize with a Sobol sequence,
  # covering the hyper-rectangle defined by our parameter limits.
  halton_sequence = tfp.mcmc.sample_halton_sequence(
      dim=6, num_results=num_particles * num_batches, seed=seed)
  halton_sequence = tf.reshape(
      halton_sequence, [num_particles, num_batches, 6])
  halton_sequences = tf.nest.pack_sequence_as(
      PARAMETER_LOWER_BOUNDS, tf.split(
          halton_sequence, num_or_size_splits=6, axis=-1))
  def interpolate(minval, maxval, h):
    return (maxval - minval) * h + minval
  return tf.nest.map_structure(
      interpolate,
      PARAMETER_LOWER_BOUNDS, PARAMETER_UPPER_BOUNDS, halton_sequences)

def update_params(num_particles, num_batches,
                  prev_params, parameter_variance, seed):
  """Update the global parameters between EAKF runs.

  Args:
    num_particles: `int` giving the number of particles for the EAKF.
    num_batches: `int` giving the number of independent EAKF runs to
      initialize in a vectorized batch.
    prev_params: A `ModelParams` tuple of the parameters used for the previous
      EAKF run.
    parameter_variance: A `ModelParams` tuple specifying how much to drift
      each parameter.
    seed: PRNG entropy.

  Returns:
    params: A `ModelParams` tuple with fields Tensors of shape
      [num_particles, num_batches] giving the global parameters
      to use for the next batch of EAKF runs.
  """
  # Initialize near the previous set of parameters. This is the first step
  # in Iterated Filtering.
  seeds = tf.nest.pack_sequence_as(
      prev_params, samplers.split_seed(seed, n=len(prev_params)))
  return tf.nest.map_structure(
      lambda x, v, seed: x + tf.math.sqrt(v) * tf.random.stateless_normal([
          num_particles, num_batches, 1], seed=seed),
      prev_params, parameter_variance, seeds)_____no_output_____

## Delays

One of the important features of this model is taking explicit account of the fact that infections are reported later than they begin.  That is, we expect that a person who moves from the $E$ compartment to the $I^r$ compartment on day $t$ may not show up in the observable reported case counts until some later day.

We assume the delay is gamma-distributed.  Following Li et al, we use 1.85 for the shape, and parameterize the rate to produce an average reporting delay of 9 days._____no_output_____

def raw_reporting_delay_distribution(gamma_shape=1.85, reporting_delay=9.):
  return tfp.distributions.Gamma(
      concentration=gamma_shape, rate=gamma_shape / reporting_delay)_____no_output_____

Our observations are discrete, so we will round the raw (continuous) delays up to the nearest day.  We also have a finite data horizon, so the delay distribution for a single person is a categorical over the remaining days.  We can therefore compute the per-city predicted observations more efficiently than sampling $O(I^r)$ gammas, by pre-computing multinomial delay probabilities instead._____no_output_____

def reporting_delay_probs(num_timesteps, gamma_shape=1.85, reporting_delay=9.):
  gamma_dist = raw_reporting_delay_distribution(gamma_shape, reporting_delay)
  multinomial_probs = [gamma_dist.cdf(1.)]
  for k in range(2, num_timesteps + 1):
    multinomial_probs.append(gamma_dist.cdf(k) - gamma_dist.cdf(k - 1))
  # For samples that are larger than T.
  multinomial_probs.append(gamma_dist.survival_function(num_timesteps))
  multinomial_probs = tf.stack(multinomial_probs)
  return multinomial_probs_____no_output_____

Here's the code for actually applying these delays to the new daily documented infectious counts:_____no_output_____

def delay_reporting(
    daily_new_documented_infectious, num_timesteps, t, multinomial_probs, seed):
  # This is the distribution of observed infectious counts from the current
  # timestep.

  raw_delays = tfd.Multinomial(
      total_count=daily_new_documented_infectious,
      probs=multinomial_probs).sample(seed=seed)

  # The last bucket is used for samples that are out of range of T + 1. Thus
  # they are not going to be observable in this model.
  clipped_delays = raw_delays[..., :-1]

  # We can also remove counts that are such that t + i >= T.
  clipped_delays = clipped_delays[..., :num_timesteps - t]
  # We finally shift everything by t. That means prepending with zeros.
  return tf.concat([
      tf.zeros(
          tf.concat([
              tf.shape(clipped_delays)[:-1], [t]], axis=0),
          dtype=clipped_delays.dtype),
      clipped_delays], axis=-1)_____no_output_____

## Inference_____no_output_____First we'll define some data structures for inference.

In particular, we'll be wanting to do Iterated Filtering, which packages the
state and parameters together while doing inference. So we'll define
a `ParameterStatePair` object.

We also want to package any side information to the model._____no_output_____

ParameterStatePair = collections.namedtuple(
    'ParameterStatePair', ['state', 'params'])

# Info that is tracked and mutated but should not have inference performed over.
SideInfo = collections.namedtuple(
    'SideInfo', [
        # Observations at every time step.
        'observations_over_time',
        'initial_population',
        'mobility_matrix_over_time',
        'population',
        # Used for variance of measured observations.
        'actual_reported_cases',
        # Pre-computed buckets for the multinomial distribution.
        'multinomial_probs',
        'seed',
    ])

# Cities can not fall below this fraction of people
MINIMUM_CITY_FRACTION = 0.6

# How much to inflate the covariance by.
INFLATION_FACTOR = 1.1

INFLATE_FN = tfes.inflate_by_scaled_identity_fn(INFLATION_FACTOR)_____no_output_____

Here is the complete observation model, packaged for the Ensemble Kalman Filter.

The interesting feature is the reporting delays (computed as previously).  The upstream model emits the `daily_new_documented_infectious` for each city at each time step._____no_output_____

# We observe the observed infections.
def observation_fn(t, state_params, extra):
  """Generate reported cases.

  Args:
    state_params: A `ParameterStatePair` giving the current parameters
      and state.
    t: Integer giving the current time.
    extra: A `SideInfo` carrying auxiliary information.

  Returns:
    observations: A Tensor of predicted observables, namely new cases
      per city at time `t`.
    extra: Update `SideInfo`.
  """
  # Undo padding introduced in `inference`.
  daily_new_documented_infectious = state_params.state.daily_new_documented_infectious[..., 0]
  # Number of people that we have already committed to become
  # observed infectious over time.
  # shape: batch + [num_particles, num_cities, time]
  observations_over_time = extra.observations_over_time
  num_timesteps = observations_over_time.shape[-1]

  seed, new_seed = samplers.split_seed(extra.seed, salt='reporting delay')

  daily_delayed_counts = delay_reporting(
      daily_new_documented_infectious, num_timesteps, t,
      extra.multinomial_probs, seed)
  observations_over_time = observations_over_time + daily_delayed_counts

  extra = extra._replace(
      observations_over_time=observations_over_time,
      seed=new_seed)

  # Actual predicted new cases, re-padded.
  adjusted_observations = observations_over_time[..., t][..., tf.newaxis]
  # Finally observations have variance that is a function of the true observations:
  return tfd.MultivariateNormalDiag(
      loc=adjusted_observations,
      scale_diag=tf.math.maximum(
          2., extra.actual_reported_cases[..., t][..., tf.newaxis] / 2.)), extra_____no_output_____

Here we define the transition dynamics. We've done the semantic work already; here we just package it for the EAKF framework, and, following Li et al, clip city populations to prevent them from getting too small._____no_output_____

def transition_fn(t, state_params, extra):
  """SEIR dynamics.

  Args:
    state_params: A `ParameterStatePair` giving the current parameters
      and state.
    t: Integer giving the current time.
    extra: A `SideInfo` carrying auxiliary information.

  Returns:
    state_params: A `ParameterStatePair` predicted for the next time step.
    extra: Updated `SideInfo`.
  """
  mobility_t = extra.mobility_matrix_over_time[..., t]
  new_seed, rk4_seed = samplers.split_seed(extra.seed, salt='Transition')
  new_state = rk4_one_step(
      state_params.state,
      extra.population,
      mobility_t,
      state_params.params,
      seed=rk4_seed)

  # Make sure population doesn't go below MINIMUM_CITY_FRACTION.
  new_population = (
      extra.population + state_params.params.intercity_underreporting_factor * (
          # Inflow
          tf.reduce_sum(mobility_t, axis=-2) -
          # Outflow
          tf.reduce_sum(mobility_t, axis=-1)))
  new_population = tf.where(
      new_population < MINIMUM_CITY_FRACTION * extra.initial_population,
      extra.initial_population * MINIMUM_CITY_FRACTION,
      new_population)

  extra = extra._replace(population=new_population, seed=new_seed)

  # The Ensemble Kalman Filter code expects the transition function to return a distribution.
  # As the dynamics and noise are encapsulated above, we construct a `JointDistribution` that when
  # sampled, returns the values above.

  new_state = tfd.JointDistributionNamed(
      model=tf.nest.map_structure(lambda x: tfd.VectorDeterministic(x), new_state))
  params = tfd.JointDistributionNamed(
      model=tf.nest.map_structure(lambda x: tfd.VectorDeterministic(x), state_params.params))

  state_params = tfd.JointDistributionNamed(
      model=ParameterStatePair(state=new_state, params=params))

  return state_params, extra_____no_output_____

Finally we define the inference method. This is two loops, the outer loop
being Iterated Filtering while the inner loop is Ensemble Adjustment Kalman Filtering._____no_output_____

# Use tf.function to speed up EAKF prediction and updates.
ensemble_kalman_filter_predict = tf.function(
    tfes.ensemble_kalman_filter_predict, autograph=False)
ensemble_adjustment_kalman_filter_update = tf.function(
    tfes.ensemble_adjustment_kalman_filter_update, autograph=False)

def inference(
    num_ensembles,
    num_batches,
    num_iterations,
    actual_reported_cases,
    mobility_matrix_over_time,
    seed=None,
    # This is how much to reduce the variance by in every iterative
    # filtering step.
    variance_shrinkage_factor=0.9,
    # Days before infection is reported.
    reporting_delay=9.,
    # Shape parameter of Gamma distribution.
    gamma_shape_parameter=1.85):
  """Inference for the Shaman, et al. model.

  Args:
    num_ensembles: Number of particles to use for EAKF.
    num_batches: Number of batches of IF-EAKF to run.
    num_iterations: Number of iterations to run iterative filtering.
    actual_reported_cases: `Tensor` of shape `[L, T]` where `L` is the number
      of cities, and `T` is the timesteps.
    mobility_matrix_over_time: `Tensor` of shape `[L, L, T]` which specifies the
      mobility between locations over time.
    variance_shrinkage_factor: Python `float`. How much to reduce the
      variance each iteration of iterated filtering.
    reporting_delay: Python `float`. How many days before the infection
      is reported.
    gamma_shape_parameter: Python `float`. Shape parameter of Gamma distribution
      of reporting delays.

  Returns:
    result: A `ModelParams` with fields Tensors of shape [num_batches],
      containing the inferred parameters at the final iteration.
  """
  print('Starting inference.')
  num_timesteps = actual_reported_cases.shape[-1]
  params_per_iter = []

  multinomial_probs = reporting_delay_probs(
      num_timesteps, gamma_shape_parameter, reporting_delay)

  seed = samplers.sanitize_seed(seed, salt='Inference')

  for i in range(num_iterations):
    start_if_time = time.time()
    seeds = samplers.split_seed(seed, n=4, salt='Initialize')
    if params_per_iter:
      parameter_variance = tf.nest.map_structure(
          lambda minval, maxval: variance_shrinkage_factor ** (
              2 * i) * (maxval - minval) ** 2 / 4.,
          PARAMETER_LOWER_BOUNDS, PARAMETER_UPPER_BOUNDS)
      params_t = update_params(
          num_ensembles,
          num_batches,
          prev_params=params_per_iter[-1],
          parameter_variance=parameter_variance,
          seed=seeds.pop())
    else:
      params_t = initialize_params(num_ensembles, num_batches, seed=seeds.pop())

    state_t = initialize_state(num_ensembles, num_batches, seed=seeds.pop())
    population_t = sum(x for x in state_t)
    observations_over_time = tf.zeros(
        [num_ensembles,
         num_batches,
         actual_reported_cases.shape[0], num_timesteps])

    extra = SideInfo(
        observations_over_time=observations_over_time,
        initial_population=tf.identity(population_t),
        mobility_matrix_over_time=mobility_matrix_over_time,
        population=population_t,
        multinomial_probs=multinomial_probs,
        actual_reported_cases=actual_reported_cases,
        seed=seeds.pop())

    # Clip states
    state_t = clip_state(state_t, population_t)
    params_t = clip_params(params_t, seed=seeds.pop())

    # Accrue the parameter over time. We'll be averaging that
    # and using that as our MLE estimate.
    params_over_time = tf.nest.map_structure(
        lambda x: tf.identity(x), params_t)

    state_params = ParameterStatePair(state=state_t, params=params_t)

    eakf_state = tfes.EnsembleKalmanFilterState(
        step=tf.constant(0), particles=state_params, extra=extra)

    for j in range(num_timesteps):
      seeds = samplers.split_seed(eakf_state.extra.seed, n=3)

      extra = extra._replace(seed=seeds.pop())

      # Predict step.

      # Inflate and clip.
      new_particles = INFLATE_FN(eakf_state.particles)
      state_t = clip_state(new_particles.state, eakf_state.extra.population)
      params_t = clip_params(new_particles.params, seed=seeds.pop())
      eakf_state = eakf_state._replace(
          particles=ParameterStatePair(params=params_t, state=state_t))

      eakf_predict_state = ensemble_kalman_filter_predict(eakf_state, transition_fn)

      # Clip the state and particles.
      state_params = eakf_predict_state.particles
      state_t = clip_state(
          state_params.state, eakf_predict_state.extra.population)
      state_params = ParameterStatePair(state=state_t, params=state_params.params)

      # We preprocess the state and parameters by affixing a 1 dimension. This is because for
      # inference, we treat each city as independent. We could also introduce localization by
      # considering cities that are adjacent.
      state_params = tf.nest.map_structure(lambda x: x[..., tf.newaxis], state_params)
      eakf_predict_state = eakf_predict_state._replace(particles=state_params)

      # Update step.

      eakf_update_state = ensemble_adjustment_kalman_filter_update(
          eakf_predict_state,
          actual_reported_cases[..., j][..., tf.newaxis],
          observation_fn)

      state_params = tf.nest.map_structure(
          lambda x: x[..., 0], eakf_update_state.particles)

      # Clip to ensure parameters / state are well constrained.
      state_t = clip_state(
          state_params.state, eakf_update_state.extra.population)

      # Finally for the parameters, we should reduce over all updates. We get
      # an extra dimension back so let's do that.
      params_t = tf.nest.map_structure(
          lambda x, y: x + tf.reduce_sum(y[..., tf.newaxis] - x, axis=-2, keepdims=True),
          eakf_predict_state.particles.params, state_params.params)
      params_t = clip_params(params_t, seed=seeds.pop())
      params_t = tf.nest.map_structure(lambda x: x[..., 0], params_t)

      state_params = ParameterStatePair(state=state_t, params=params_t)
      eakf_state = eakf_update_state
      eakf_state = eakf_state._replace(particles=state_params)

      # Flatten and collect the inferred parameter at time step t.
      params_over_time = tf.nest.map_structure(
          lambda s, x: tf.concat([s, x], axis=-1), params_over_time, params_t)

    est_params = tf.nest.map_structure(
        # Take the average over the Ensemble and over time.
        lambda x: tf.math.reduce_mean(x, axis=[0, -1])[..., tf.newaxis],
        params_over_time)
    params_per_iter.append(est_params)
    print('Iterated Filtering {} / {} Ran in: {:.2f} seconds'.format(
        i, num_iterations, time.time() - start_if_time))

  return tf.nest.map_structure(
      lambda x: tf.squeeze(x, axis=-1), params_per_iter[-1])_____no_output_____

Final detail: clipping the parameters and state consists of making sure they are within range, and non-negative._____no_output_____

def clip_state(state, population):
  """Clip state to sensible values."""
  state = tf.nest.map_structure(
      lambda x: tf.where(x < 0, 0., x), state)

  # If S > population, then adjust as well.
  susceptible = tf.where(state.susceptible > population, population, state.susceptible)
  return SEIRComponents(
      susceptible=susceptible,
      exposed=state.exposed,
      documented_infectious=state.documented_infectious,
      undocumented_infectious=state.undocumented_infectious,
      daily_new_documented_infectious=state.daily_new_documented_infectious)

def clip_params(params, seed):
  """Clip parameters to bounds."""
  def _clip(p, minval, maxval):
    return tf.where(
        p < minval,
        minval * (1. + 0.1 * tf.random.stateless_uniform(p.shape, seed=seed)),
        tf.where(p > maxval,
                 maxval * (1. - 0.1 * tf.random.stateless_uniform(
                     p.shape, seed=seed)), p))
  params = tf.nest.map_structure(
      _clip, params, PARAMETER_LOWER_BOUNDS, PARAMETER_UPPER_BOUNDS)

  return params_____no_output_____

## Running it all together_____no_output_____

# Let's sample the parameters.
#
# NOTE: Li et al. run inference 1000 times, which would take a few hours.
# Here we run inference 30 times (in a single, vectorized batch).
best_parameters = inference(
    num_ensembles=300,
    num_batches=30,
    num_iterations=10,
    actual_reported_cases=observed_daily_infectious_count,
    mobility_matrix_over_time=mobility_matrix_over_time)Starting inference.
Iterated Filtering 0 / 10 Ran in: 26.65 seconds
Iterated Filtering 1 / 10 Ran in: 28.69 seconds
Iterated Filtering 2 / 10 Ran in: 28.06 seconds
Iterated Filtering 3 / 10 Ran in: 28.48 seconds
Iterated Filtering 4 / 10 Ran in: 28.57 seconds
Iterated Filtering 5 / 10 Ran in: 28.35 seconds
Iterated Filtering 6 / 10 Ran in: 28.35 seconds
Iterated Filtering 7 / 10 Ran in: 28.19 seconds
Iterated Filtering 8 / 10 Ran in: 28.58 seconds
Iterated Filtering 9 / 10 Ran in: 28.23 seconds

The results of our inferences.  We plot the maximum-likelihood values for all the global paramters to show their variation across our `num_batches` independent runs of inference.  This corresponds to Table S1 in the supplemental materials._____no_output_____

fig, axs = plt.subplots(2, 3)
axs[0, 0].boxplot(best_parameters.documented_infectious_tx_rate,
                  whis=(2.5,97.5), sym='')
axs[0, 0].set_title(r'$\beta$')

axs[0, 1].boxplot(best_parameters.undocumented_infectious_tx_relative_rate,
                  whis=(2.5,97.5), sym='')
axs[0, 1].set_title(r'$\mu$')

axs[0, 2].boxplot(best_parameters.intercity_underreporting_factor,
                  whis=(2.5,97.5), sym='')
axs[0, 2].set_title(r'$\theta$')

axs[1, 0].boxplot(best_parameters.average_latency_period,
                  whis=(2.5,97.5), sym='')
axs[1, 0].set_title(r'$Z$')

axs[1, 1].boxplot(best_parameters.fraction_of_documented_infections,
                  whis=(2.5,97.5), sym='')
axs[1, 1].set_title(r'$\alpha$')

axs[1, 2].boxplot(best_parameters.average_infection_duration,
                  whis=(2.5,97.5), sym='')
axs[1, 2].set_title(r'$D$')
plt.tight_layout()_____no_output_____


# Notebook from mahdi-b/ICS434-Sp2022
Path: morea/Data_Wrangling_and_Pandas_2/resources/9_group_by.ipynb

# from IPython.core.display import HTML
import pandas as pd _____no_output_____spending_df = pd.read_csv('https://www.dropbox.com/s/ce9b47nzt3sx7y5/spending_10k.csv?dl=1', index_col="unique_id", dtype={"doctor_id":"object"})
spending_df.head(10)_____no_output_____

### Overview

* In this section, we will tackle the handy `groupby` method.

* We also cover the split-apply-combine scheme to:

  * Aggregate data in each group
  * Transform data in each group
  * Filter the data in each group
  * Thin the data in each group_____no_output_____### `group_by` and `DataFrame` groups

* The `groupby()` method is used to group the data using values from one or more columns.

   * `groupby` takes as input one or more column labels, which it uses to group the data.

```python
df_1.groupby("X")
```

![](https://www.dropbox.com/s/86bi697t59zmkdn/groupby.png?dl=1)

_____no_output_____### Identifying Groups from a GroupBy Object

```python
spending_df.groupby('specialty')
```

![](https://www.dropbox.com/s/bs8o34e4s7bdqa8/group_by_specialty.png?dl=1)

* The `groupby` method returns an object of type `DataFrameGroupBy.`
  * This is not a `DataFrame`, and does not, therefore, have the `DataFrame` methods discussed previously

* We can get `DataFrameGroupBy`  element using a valid `groupby` column(s) value(s)
_____no_output_____

x = spending_df.groupby('specialty')
type(x)_____no_output_____spending_by_specialty = spending_df.groupby('specialty')

addiction_med_group = spending_by_specialty.get_group("ADDICTION MEDICINE")
addiction_med_group_____no_output_____

### `groupby` and Group-Specific Processing

* An ideal use-case for `groupby` consists of applying operations to each group independently.

* For instance, to compute the total spending by `specialty`, we need to:
  * Split the data by `specialty`.
  * Sum the total `spending` for each type of `specialty`
  * Combine and return the sums for each group into as a new `DataFrame`.

_____no_output_____
### Split-Apply-Combine Paradigm

* `groupby()` is often applied in the context of the data processing paradigm called "split-apply-combine".

  * **Split**: you need to split the data into chunks defined using one or more columns.
    * This is typically done using `groupby`.
  * **Apply**: apply some operation to the chunks generated.
    * Ex. Count the number of rows in each chunk, average the values for a specific column, etc.
  * **Combine**: combine the results of the applied operation into a new `DataFrame`.

_____no_output_____### Split-Apply-Combine Example

![](https://www.dropbox.com/s/aecufw3mfu2mlah/split_apply_combine_example.png?dl=1)

* The type of Split-Apply-Combine applied here is referred to as aggregation.
  * Aggregations refer to any operation that aggregates (reduces) group data to a single value._____no_output_____### The 3 ( or 3  $\frac{1}{2}$) Classes of Opearations on Groups

* Three are 3 formal classes of split-apply-combine operations that can be applied to group data.

  * I include a variant ($\frac{1}{2}$ a class) which I think is useful to better classify split-apply-combine operations.

1\.$~~$__Aggregations__ generate a single value for each group
   * E.g., Sum the spending by specialty

2\.$~~$ __Transformations__ convert the data and generate a group of the same size as the original group.
   * E.g., Convert the currency by country for some datasets that contains medication cost by country.

3\.$~~$ __Filters__ retain or discard a group based on group-specific boolean computations.
   * E.g., drop specialty if the sum of spending is below some threshold

3$\frac{1}{2}$\.$~$"__Thinning__" drops entries in a group based on some defined logic.
  * E.g., compute how far off are the group values from the group mean._____no_output_____### Aggregations

- __Aggregations__ aggregate the data in each group, i.e., they reduce the data in each group to a single value.

  * This includes, for instance, computing group sums, means, maximums, minimums, _etc_.

_____no_output_____![](https://www.dropbox.com/s/9q54na9szs5syi5/aggregate.png?dl=1)
_____no_output_____
### Transforming Group Data

* __Transform__ will trnsform the data in a group-specific way.

   * E.g., transform the value by dividing it by the group mean

*  The number of entries per group resulting from a transformation is the same as the number of entries in the group before the transformation.

_____no_output_____- The diagram below shows an example where the data in column "Y" in transformed by dividing it by the group mean.

![](https://www.dropbox.com/s/nf8lg0lqk3yxf7k/transform_2.png?dl=1)
_____no_output_____### Filtering Group Data

* Consist of dropping or retaining that group in a way that depends on a group-specific computation that returns `True` or `False`.

* For instance, we can filter specialties that don't have enough entries or for which the mean `spending` if below a certain threshold.
  * Groups are either retained or discarded. Groups that are retained are unmodified.
_____no_output_____- The diagram below shows an example where groups are filtered if their sum for column `Y` is less than 10.

![](https://www.dropbox.com/s/ncmv2xsupjok7va/filter.png?dl=1)_____no_output_____### Thinning Group Data

* Consist of reducing the number of entries using a group-specific operation.

* Thinning can be useful to, for instance:
  * Sub-sample the data at the group level (create a blanced dataset)
  * Returning the top `n` entries in each group
  * etc.

  * As opposed to aggregating functions, thinning does not have to reduce the group into a single entry; although it could

![](https://www.dropbox.com/s/m4p4f5nk55w2ni2/thin.png?dl=1) _____no_output_____
### Aggregating the Data Using `groupby`

* Aggregation is commonly used to compute summary statistics on each of the groups.

* Some of the interesting/important summary aggregation methods `DataFrameGroupBy` objects are:

|Methods           |        Decription                              |
|:-----------------|:-----------------------------------------------|
| `mean`, `median` | Computes the mean and the median in each group |
| `min` , `max`    | computes the min and max in each group         |
| `size`           | computes the number of values in each group    |

* The function applies to all valid columns that are available after using the `groupby` argument as an index

_____no_output_____##### Aggregating the Data Using `groupby` Cont'd

- The functions above all use the same syntax:

```python
spending_df.groupby('specialty').sum()
# or
spending_df.groupby('specialty').min()
```
_____no_output_____

df = pd.DataFrame({"A":[1,1,2,2,3,3], "B":[1,2,3,4,5,6], "C":["A","B","C","D","E","F"]})

df.groupby("A").sum()
_____no_output_____spending_df.head()_____no_output_____spending_df.groupby('specialty').sum().head(10)
_____no_output_____

### Applying Functions to Group Columns

* The method `agg()` can be used where complex or custom aggregation logic is required.
* Note tht `agg` applies to a `DataFrameGroupBy` object
  *  The method `agg` takes a function (or a list of functions) and uses it (them) to aggregate the group's colum(s)

* Example, we can use `sum_spending_CAD` to return the sum of the spending in Canadian Dollars.
_____no_output_____```python
def sum_spending_CAD(x):
    return x.sum() * 1.32

spending_by_specialty['spending'].agg(sum_spending_CAD)
```

* `agg` can take a dictionary of functions to aggregate on.
    * Required for aggregating more than one column

    ```python
    spending_by_specialty.agg({'nb_beneficiaries' :sum,
                               'spending' : sum_spending_CAD)
    ```

* `agg` can take a list of function to apply to each column functions to aggregate on.

    `
    spending_by_specialty.agg([min,max,sum])
    `_____no_output_____

def sum_spending_CAD(x):
    return x.sum() * 1.27

# format
(
    spending_by_specialty.agg({ 'nb_beneficiaries': sum, 'spending': sum_spending_CAD })
                         .head(5)
)

_____no_output_____# note that sum, min and max here are functions
spending_by_specialty[['spending', 'nb_beneficiaries']].agg([sum, min, max]).head()_____no_output_____spending_by_specialty.agg({'nb_beneficiaries' :min,
                           'spending' : max}).head()
_____no_output_____spending_by_specialty.agg({'nb_beneficiaries' :[min, sum],
                           'spending' : max}).head()
_____no_output_____

### Transforming  the `groupby` `DataFrames`

- As opposed to aggregations, which reduce the data into a single value, transformations modify the data but don't change the `shape` (dimension) of the groups

- Transformations are useful for applying operations that are group specific_____no_output_____### Transforming the Data in `groupby` Cont'd

- The example below computes the percent contribution of each entry to each specialty by applying a transformation that normalizes the entry's spending over the total spending in that specialty.

![](https://www.dropbox.com/s/ewfaj5rjszbvzz3/transform_spending.png?dl=1)
_____no_output_____

spending_by_specialty["spending"].get_group("ADDICTION MEDICINE")_____no_output_____

### Applying a Transformation

- Applying a transformation is done using the method called `transform`.

- The method `transform` takes as input a function name, which it calls on each group of the `DataFrameGroupBy` object_____no_output_____

def my_function(x):
    return (x   / x.sum() ) * 100

spending_by_specialty['spending'].transform(my_function).head(10)

_____no_output_____spending_df["spending_pct"] = spending_by_specialty['spending'].transform(my_function)

spending_df.head()_____no_output_____spending_df[spending_df['specialty'] == "ADDICTION MEDICINE"]_____no_output_____spending_df.sort_values(['specialty', 'spending_pct'], ascending=[True, False]).head(10)_____no_output_____

### More complex Transformations

* As you may have noted above, drugs are still duplicated across `doctor_ids` within the same `specialty.`
  *  ex. FLUTICASONE/SALMETEROL is prescribed by at least 3 doctors whose specialty is ALLERGY/IMMUNOLOGY

* To see the percent spending by `drug` column, we need to group on both the `specialty` and the `medication` and then sum the `spending_pct` computed previously

```python
medication_spendng_pct =  spending_df.groupby(["specialty", "medication"])["spending_pct"].sum()
```

_____no_output_____

medication_spendng_pct =  spending_df.groupby(["specialty", "medication"])["spending_pct"].sum()
medication_spendng_pct.head()_____no_output_____

medication_spendng_pct.index

_____no_output_____

### Pandas `MultiIndex`

* When we group by more than one column, the result is a (Multindex).
  *  I.e., the index that involves more than one column
* Indexing into a 'Series' or DataFrame that has a MultiIndex requires a tuple with at least one value

https://www.dropbox.com/s/tuzw74t017vyk9m/multi-index.png?dl=1_____no_output_____

df = pd.DataFrame([[1,1,1], [1,2,2], [2,1,3], [2,2,4]], columns=["A", "B", "C"])
df
_____no_output_____df = df.set_index(["A", "B"])
df.index
_____no_output_____medication_spendng_pct[('ADDICTION MEDICINE', 'BUSPIRONE HCL')]_____no_output_____medication_spendng_pct[('ADDICTION MEDICINE', )]_____no_output_____medication_spendng_pct[('ALLERGY/IMMUNOLOGY', )]_____no_output_____medication_spendng_pct[('ALLERGY/IMMUNOLOGY', )].sum()_____no_output_____

### More complex Transformations - Cont'd

* The multi-index is sometimes inconvenient to work with, or we may have a better column to index on

* We can reset (drop) the index using the method `reset_index`
  * `reset_index` converts the MutliIndex into columns and uses a range index (0 to n-1)
  * Allows us to sort on `specialty` and `spending_pct` as we did earlier

_____no_output_____

spending_df.groupby(["specialty", "medication"])["spending_pct"].sum().head()_____no_output_____medication_spendng_pct = spending_df.groupby(["specialty", "medication"])["spending_pct"].sum().reset_index()
medication_spendng_pct.head()
_____no_output_____medication_spendng_pct.sort_values(["specialty", "spending_pct"], ascending=[True, False]).head(10)_____no_output_____

### Filtering Groups

- Filtering a group is done using the method called `filter`

- The method `filter` takes as input a function name, which it calls on each group of the `DataFrameGroupBy` object
  - The function must return either `True` or `False`.
  - Groups for which the function returns `False` are dropped.

- The resulting` DataFrame` has its entries in the same order as the original `DataFrame`.

_____no_output_____

df = pd.DataFrame([[1,1], [1,2], [2,3], [2,4]], columns=["A", "B"])
df
_____no_output_____df.groupby("A").filter(lambda x: x["B"].sum() > 4)_____no_output_____print(len(spending_df['specialty'].unique()))
spending_df['specialty'].unique()[:20]75

def filter_on_spending(x):
    return x['spending'].sum() > 50_000

high_spending_df = spending_df[["specialty", 'spending']].groupby('specialty').filter(filter_on_spending)

_____no_output_____print(len(high_spending_df['specialty'].unique()))
high_spending_df['specialty'].unique()[:20]
38

### Thinning Groups

* Thinning the data consist in reducing the number of entries in a group

* As opposed to aggregating functions, thinning does not have to reduce the group into a single entry
  * Although it could reduce it to a single entry

* Thinning can be used, for instance, to return only the top 3 entries in each category, or to randomly sample a small subset of entries from each category_____no_output_____### Thinning Methods and `apply`

* `pandas` offers a few methods for thinning the data
  * Ex. `nlargest`, `nsmallest`

* However, thinning  is most often carried out using a method  called `apply`

* The  method `apply` takes as input a function that it calls on each group of the `DataFrameGroupBy` object
_____no_output_____

spending_by_specialty['spending'].nlargest(2)_____no_output_____spending_by_specialty['spending'].nsmallest(3)_____no_output_____import pandas as pd
data_frame = pd.DataFrame({"A": [1,2,3,4]})
data_frame_____no_output_____# temp
data_frame = pd.DataFrame({"A": [1,2,3,4]})

def some_function(x):
    print(type(x))
    print(x)
    print("----")
    # does somet
    return x**1.4

data_frame.apply(some_function, axis=0)<class 'pandas.core.series.Series'>
0    1
1    2
2    3
3    4
Name: A, dtype: int64
----
# implementation of nlargest using apply
spending_by_specialty['spending'].apply(lambda x: x.sort_values(ascending=False).head(3))_____no_output_____

### Sub-sampling a `DataFrame`

* Subsampling is the process of selecting a subset of the original data.

* This can be achived using the DataFrame mthod called `sample.`
  * On of two parameters are relevant in this scenario,`n` the number of samples to randomly select or `frac` a portion of the data to retun

```python
 spending_df.sample(frac=0.001)
```

* Subsampling is an ideal way to handle datasets that are unbalanced across some factor(s).

_____no_output_____

# return 0.01% of the data, i.e 100 entries
spending_df.sample(frac=0.01)
_____no_output_____spending_df.sample(n=10) _____no_output_____spending_df.groupby('specialty').size()_____no_output_____spending_df.groupby('specialty').size().plot.bar(figsize=(16,4))_____no_output_____# We sample only 10% of the Data in each category

def sample_10p(x):
    return x.sample(n=10, replace=True)

# spending_by_specialty.apply(sample_10p).head()
spending_subsampled_df = spending_df.groupby('specialty').apply(sample_10p)_____no_output_____spending_subsampled_df.head()_____no_output_____spending_subsampled_df.drop("specialty", axis =1 )_____no_output_____# Note that we first need to drop the colum
spending_subsampled_df = spending_subsampled_df.drop("specialty", axis =1).reset_index((0,))
spending_subsampled_df_____no_output_____spending_subsampled_df.groupby('specialty').size().plot.bar(figsize=(16,4))_____no_output_____