# Notebook from r-peng/quimb
Path: docs/examples/ex_tn_train_circuit.ipynb

%config InlineBackend.figure_formats = ['svg']
import quimb as qu
import quimb.tensor as qtn_____no_output_____

def single_qubit_layer(circ, gate_round=None):
    """Apply a parametrizable layer of single qubit ``U3`` gates.
    """
    for i in range(circ.N):
        # initialize with random parameters
        params = qu.randn(3, dist='uniform')
        circ.apply_gate(
            'U3', *params, i,
            gate_round=gate_round, parametrize=True)

def two_qubit_layer(circ, gate2='CZ', reverse=False, gate_round=None):
    """Apply a layer of constant entangling gates.
    """
    regs = range(0, circ.N - 1)
    if reverse:
        regs = reversed(regs)

    for i in regs:
        circ.apply_gate(
            gate2, i, i + 1, gate_round=gate_round)

def ansatz_circuit(n, depth, gate2='CZ', **kwargs):
    """Construct a circuit of single qubit and entangling layers.
    """
    circ = qtn.Circuit(n, **kwargs)

    for r in range(depth):
        # single qubit gate layer
        single_qubit_layer(circ, gate_round=r)

        # alternate between forward and backward CZ layers
        two_qubit_layer(
            circ, gate2=gate2, gate_round=r, reverse=r % 2 == 0)

    # add a final single qubit layer
    single_qubit_layer(circ, gate_round=r + 1)

    return circ_____no_output_____

n = 6
depth = 9
gate2 = 'CZ'

circ = ansatz_circuit(n, depth, gate2=gate2)
circ_____no_output_____

We can extract just the unitary part of the circuit as a tensor network like so:_____no_output_____

V = circ.uni_____no_output_____

You can see it already has various ``tags`` simulteneously identifying different structures:_____no_output_____

# types of gate
V.draw(color=['U3', gate2], show_inds=True)_____no_output_____# layers of gates
V.draw(color=[f'ROUND_{i}' for i in range(depth + 1)], show_inds=True)_____no_output_____# what register each tensor is 'above'
V.draw(color=[f'I{i}' for i in range(n)], show_inds=True)_____no_output_____# a unique tag for per gate applied
V.draw(color=[f'GATE_{i}' for i in range(circ.num_gates)], legend=False)_____no_output_____

# the hamiltonian
H = qu.ham_ising(n, jz=1.0, bx=0.7, cyclic=False)

# the propagator for the hamiltonian
t = 2
U_dense = qu.expm(-1j * t * H)

# 'tensorized' version of the unitary propagator
U = qtn.Tensor(
    data=U_dense.reshape([2] * (2 * n)),
    inds=[f'k{i}' for i in range(n)] + [f'b{i}' for i in range(n)],
    tags={'U_TARGET'}
)
U.draw(color=['U3', gate2, 'U_TARGET'])_____no_output_____

The core object describing how similar two unitaries are is: $\mathrm{Tr}(V^{\dagger}U)$, which we can naturally visualize at a tensor network:_____no_output_____

(V.H & U).draw(color=['U3', gate2, 'U_TARGET'])_____no_output_____

For our loss function we'll normalize this and negate it (since the optimizer *minimizes*)._____no_output_____

def loss(V, U):
    return 1 - abs((V.H & U).contract(all, optimize='auto-hq')) / 2**n

# check our current unitary 'infidelity':
loss(V, U)_____no_output_____

# use the autograd/jax based optimizer

tnopt = qtn.TNOptimizer(
    V,                        # the tensor network we want to optimize
    loss,                     # the function we want to minimize
    loss_constants={'U': U},  # supply U to the loss function as a constant TN
    tags=['U3'],              # only optimize U3 tensors
    autodiff_backend='jax',   # use 'autograd' for non-compiled optimization
    optimizer='L-BFGS-B',     # the optimization algorithm
)_____no_output_____

# allow 10 hops with 500 steps in each 'basin'
V_opt = tnopt.optimize_basinhopping(n=500, nhop=10)0.004380524158477783 [best: 0.004311561584472656] :  34%|███▍      | 1720/5000 [00:42<01:21, 40.44it/s]

The optimized tensor network still contains ``PTensor`` instances but now with optimized parameters.
For example, here's the tensor of the ``U3`` gate acting on qubit-2 in round-4:_____no_output_____

V_opt['U3', 'I2', 'ROUND_4']_____no_output_____

We can see the parameters have been updated by the training:_____no_output_____

# the initial values
V['U3', 'ROUND_4', 'I2'].params_____no_output_____# the optimized values
V_opt['U3', 'ROUND_4', 'I2'].params_____no_output_____

We can see what gate these parameters would generate:_____no_output_____

qu.U_gate(*V_opt['U3', 'ROUND_4', 'I2'].params)_____no_output_____

A final sanity check we can perform is to try evolving a random state with the target unitary and trained circuit and check the fidelity between the resulting states.

First we turn the tensor network version of $V$ into a dense matrix:_____no_output_____

V_opt_dense = V_opt.to_dense([f'k{i}' for i in range(n)], [f'b{i}' for i in range(n)])_____no_output_____

Next we create a random initial state, and evolve it with the _____no_output_____

psi0 = qu.rand_ket(2**n)

# this is the exact state we want
psif_exact = U_dense @ psi0

# this is the state our circuit will produce if fed `psi0`
psif_apprx = V_opt_dense @ psi0_____no_output_____

The (in)fidelity should broadly match our training loss:_____no_output_____

f"Fidelity: {100 * qu.fidelity(psif_apprx, psif_exact):.2f} %"_____no_output_____

circ.update_params_from(V_opt)

# the basic gate specification
circ.gates_____no_output_____


# Notebook from ebmdatalab/covid_trials_tracker-covid
Path: notebooks/ictrp_data_handling.ipynb

# Data cleaning and handling_____no_output_____

#ICTRP Search: "covid-19" or "novel coronavirus" or "2019-ncov" or "covid19" or "sars-cov-2"
import xmltodict
import pandas as pd
import numpy as np
from datetime import date
import re
import unicodedata
import text_unidecode as unidecode
from collections import Counter

#POINT THIS TO THE UPDATED XML - deprecated as ICTRP is down.

#with open('ICTRP-Results_18Mar2020.xml', 'rb') as f:
#    xml = xmltodict.parse(f, dict_constructor=dict)

#df = pd.DataFrame(xml['Trials_downloaded_from_ICTRP']['Trial'])

#This now takes the CSV posted by the ICTRP as an input from here: https://www.who.int/ictrp/en/
#One bit of pre-data management to do before loading: I take the "Date enrollment" field and in
#Excel I format it as a date, otherwise they are a pain to import due to differeing formats
#I then save it as an excel spreadsheet from the original CSV.

#df = pd.read_excel('this_weeks_data/COVID19-web_12aug2020.xlsx', dtype={'Phase': str})
df = pd.read_csv('this_weeks_data/COVID19-web_16dec2020.csv', dtype={'Phase': str})

#UPDATE THESE WITH EACH RUN
prior_extract_date = date(2020,11,11)
this_extract_date = date(2020,12,16)

def enrollment_dates(x):
    format_1 = re.compile(r"\d{4}/\d{2}/\d{2}")
    format_2 = re.compile(r"\d{2}/\d{2}/\d{4}")
    format_3 = re.compile(r"\d{4}-\d{2}-\d{2}")
    format_4 = re.compile(r"\d{2}-\d{2}-\d{4}")
    format_5 = re.compile(r"\w{3}-\d{2}")
    if isinstance(x, str) and bool(re.match(format_5, x)):
        return(pd.to_datetime(x, format='%b-%y'))
    elif isinstance(x, str) and x[0].isalpha():
        return pd.to_datetime(x)
    elif isinstance(x, str) and bool(re.match(format_1, x)):
        return pd.to_datetime(x, format='%Y/%m/%d')
    elif isinstance(x, str) and bool(re.match(format_2, x)):
        return pd.to_datetime(x, format='%d/%m/%Y')
    elif isinstance(x, str) and bool(re.match(format_3, x)):
        return pd.to_datetime(x, format='%Y-%m-%d')
    elif isinstance(x, str) and bool(re.match(format_4, x)):
        return pd.to_datetime(x, format='%d-%m-%Y')
    else:
        return pd.to_datetime(x, errors='coerce')

def fix_date(x):
    if isinstance(x,str):
        return x
    else:
        x = pd.to_datetime(x).date()
        return x

#This is fixes for known broken enrollement dates
enrollment_errors= {
    'IRCT20200310046736N1': ['14/06/2641', '2020-04-01'],
    'EUCTR2020-001909-22-FR': ['nan', '2020-04-29']
}

reg_date_errors = {
    'RBR-5p8nzk6': ['0', '20201125'],
    'RBR-8tygcz7': ['0', '20201201']
}

def fix_errors(fix_dict, df, col):
    for a, b in fix_dict.items():
        ind = df[df.TrialID == a].index.values[0]
        if str(df.at[ind, col]) == str(b[0]):
            df.at[ind, col] = b[1]
        else:
            print(f'Original Value Did not Match for {a}')
    return df

def d_c(x):
    return x[x.TrialID.duplicated()]_____no_output_____df = fix_errors(enrollment_errors, df, 'Date enrollement')

df = fix_errors(reg_date_errors, df, 'Date registration3')_____no_output_____df['Date enrollement'] = df['Date enrollement'].apply(enrollment_dates)

df['Date registration'] = pd.to_datetime(df['Date registration3'], format='%Y%m%d')

#df['Date registration'] = pd.to_datetime(df['Date registration'], format='%d/%m/%Y')
#df['Date enrollement'] = pd.to_datetime(df['Date enrollement'], errors='coerce')_____no_output_____#Extracting target enrollment
size = df['Target size'].tolist()

extracted_size = []
for s in size:
    try:
        s = int(s)
        extracted_size.append(s)
    except (ValueError, TypeError):
        if not s or pd.isnull(s):
            extracted_size.append('Not Available')
        elif isinstance(s,str):
            digits = []
            nums = re.findall(r':\d{1,10};',s)
            for n in nums:
                digits.append(int(n.replace(':','').replace(';','')))
            extracted_size.append(sum(digits))
        else:
            print(type(s))

df['target_enrollment'] = extracted_size

#Creating retrospective registration
df['retrospective_registration'] = np.where(df['Date registration'] > df['Date enrollement'], True, False)_____no_output_____#Taking only what we need right now

cols = ['TrialID', 'Source Register', 'Date registration', 'Date enrollement', 'retrospective_registration',
        'Primary sponsor', 'Recruitment Status', 'Phase', 'Study type', 'Countries', 'Public title', 'Acronym',
        'Intervention', 'target_enrollment', 'web address', 'results yes no', 'results url link',
        'Last Refreshed on']

df_cond = df[cols].reset_index(drop=True)

#renaming columns to match old format so I don't have to change them throughout
df_cond.columns = ['TrialID', 'Source_Register', 'Date_registration', 'Date_enrollement',
                   'retrospective_registration', 'Primary_sponsor', 'Recruitment_Status', 'Phase', 'Study_type',
                   'Countries', 'Public_title', 'Acronym', 'Intervention', 'target_enrollment', 'web_address',
                   'has_results', 'results_url_link', 'Last_Refreshed_on']

print(f'The ICTRP shows {len(df_cond)} trials as of {this_extract_date}')_____no_output_____#POINT THIS TO LAST WEEK'S PROCESSED DATA
last_weeks_trials = pd.read_csv('last_weeks_data/trial_list_2020-11-11.csv').drop_duplicates()_____no_output_____#Check for which registries we are dealing with:
df_cond.Source_Register.value_counts()_____no_output_____

When working with data straight from XML we need to do some tedious tidying up of dates because of different formats. They do not parse correctly by default in Pandas. Old code for this is in old commits. The code a few cells above handles this now._____no_output_____

#lets get rid of trials from before 2020 for now

pre_2020 = len(df_cond[df_cond['Date_registration'] < pd.Timestamp(2020,1,1)])

print(f'Excluded {pre_2020} trials from before 2020')

df_cond_rec = df_cond[df_cond['Date_registration'] >= pd.Timestamp(2020,1,1)].reset_index(drop=True)

print(f'{len(df_cond_rec)} trials remain')_____no_output_____

This code removes trials that indicate that they never started. This is done on the Chinese registry through specific language in the Title. Trials from ClinicalTrials.gov are indicated by the `Withdrawn` trial status.

This will be expanded moving forward to account for the unique terminology used by other registries as necessary moving forward._____no_output_____

#Removing cancelled/withdrawn trials for what registries we have to date

cancelled_trials = len(df_cond_rec[(df_cond_rec['Public_title'].str.contains('Cancelled')) |
                                   (df_cond_rec['Public_title'].str.contains('Retracted due to'))])

print(f'Excluded {cancelled_trials} cancelled trials with no enrollment via Title')

#Now lets get rid of registrations we know don't belong from manual review, cross-referenced with
#ClinicalTrials.gov list of "Withdrawn" trials. See 'manual_data.xlsx' for reasons for exclusion

exclusions = pd.read_excel('manual_data.xlsx', sheet_name = 'manual removals')

exclude = exclusions.trial_id.to_list()

#This gets all the COVID trials currently listed as "Withdrawn" on ClinicalTrials.gov

ct_gov_withdrawn = pd.read_csv('https://clinicaltrials.gov/ct2/results/download_fields??cond=COVID-19&term=&type=&rslt=&recrs=i&age_v=&gndr=&intr=&titles=&outc=&spons=&lead=&id=&cntry=&state=&city=&dist=&locn=&rsub=&strd_s=&strd_e=&prcd_s=&prcd_e=&sfpd_s=&sfpd_e=&rfpd_s=&rfpd_e=&lupd_s=&lupd_e=&sort=&down_count=1000&down_flds=all&down_fmt=csv')

ct_w = ct_gov_withdrawn['NCT Number'].to_list()

if set(ct_w) - set(exclude):
    print(f'Add new Withdrawn Trials from ClincialTrials.gov: {set(ct_w) - set(exclude)}')
else:
    print('All Withdrawn trials accounted for')_____no_output_____print(f'Excluded {len(exclude)} non-COVID trials screened through manual review')

df_cond_nc = df_cond_rec[~((df_cond_rec['Public_title'].str.contains('Cancelled')) |
                           (df_cond_rec['Public_title'].str.contains('Retracted due to'))) &
                         ~(df_cond_rec['Recruitment_Status'] == "Withdrawn") &
                         ~(df_cond_rec['TrialID'].isin(exclude))].reset_index(drop=True)

print(f'{len(df_cond_nc)} trials remain')_____no_output_____

As a general rule, simply for quality and ease of use, we will usually default to a ClinicalTrials.gov record over another type of registration in instances of cross-registration. The ICTRP alerts users to trials with known cross-registrations in the "Bridge" field of their dataset and only lists 1 registration per trial (but does not tell you the cross-registered trial IDs). These aren't comprehensive but are a good start. We can manually check and catalogue these. However we will want to replace some of these when the "Parent" registry is another registry. The first step is to remove the duplicate or replaced entries, then we will add the ClinicalTrials.gov (or another) version of the registry entry  back into the dataset when we append known trials. We will then join in a new column listing the known cross-registered trial ids._____no_output_____

c_reg = pd.read_excel('manual_data.xlsx', sheet_name = 'cross registrations')
replace_ids = c_reg.id_to_replace.tolist()

replaced = df_cond_nc[df_cond_nc.TrialID.isin(replace_ids)]
print(f'{len(replaced)} known cross registrations will be replaced')

df_cond_nc = df_cond_nc[~(df_cond_nc.TrialID.isin(replace_ids))].reset_index(drop=True)_____no_output_____#Function for checking changes

def trial_diffs(new=True):
    df = df_cond_nc.merge(last_weeks_trials['trialid'], left_on = 'TrialID', right_on = 'trialid', how='outer', indicator=True)
    if new:
        new = df[(df['_merge'] == 'left_only')]
        return new['TrialID'].tolist()
    else:
        dropped = df[(df['_merge'] == 'right_only')]
        return dropped['trialid'].tolist()_____no_output_____additions = pd.read_excel('manual_data.xlsx', sheet_name = 'additional_trials').drop('from',
                                                                                axis=1).reset_index(drop=True)

print(f'There are approximately {len(trial_diffs(new=True))} new trials')

added = additions.TrialID.tolist()

print(f'The following trials were removed since the last time and were manually checked to confirm:')
print(list(set(trial_diffs(new=False)) - set(added) - set(replace_ids)))_____no_output_____

Now we just need to take a quick look at trials that came and went since the last update. We can add in any additional trials that we know about that are not accounted for in the ICTRP database._____no_output_____

#Here we check to see if any of our manual additions have been added to the dataset

for t in added:
    if t in df_cond_nc.TrialID.tolist():
        print(f'{t} is already in the data')
    else:
        continue_____no_output_____#These are trials we know about that are not showing up in the ICTRP data pull
#or are re-added as cross-registrations

print(f'An additional {len(additions)} known trials, or preferred cross registrations were added to the data')

print(added)

df_cond_all = df_cond_nc.append(additions)
df_cond_all['Date_enrollement'] = df_cond_all['Date_enrollement'].apply(enrollment_dates)

print(f'The final dataset is {len(df_cond_all)} trials')_____no_output_____#This ensures our check for retrospective registration is accurate w/r/t cross-registrations

c_r_comp_dates = c_reg[['trial_id_keep', 'cross_reg_date']].groupby('trial_id_keep', as_index=False).min()
c_r_merged = c_r_comp_dates.merge(df_cond_nc[['TrialID', 'Date_registration', 'Date_enrollement']],
                                 left_on='trial_id_keep', right_on='TrialID', how='left')
c_r_merged['earliest_reg'] = c_r_merged[['cross_reg_date', 'Date_registration']].min(axis=1)
pre_reg = c_r_merged[c_r_merged.TrialID.notnull() & (c_r_merged.earliest_reg <= c_r_merged.Date_enrollement)].trial_id_keep.to_list()

ret_reg = c_r_merged[c_r_merged.TrialID.notnull() & ~(c_r_merged.earliest_reg <= c_r_merged.Date_enrollement)].trial_id_keep.to_list()
ret_reg

for index, row in df_cond_all.iterrows():
    if row.TrialID in pre_reg:
        df_cond_all.at[index, 'retrospective_registration'] = True
    elif row.TrialID in ret_reg:
        df_cond_all.at[index, 'retrospective_registration'] = False_____no_output_____#finally, add cross-registration field

df_cond_all = df_cond_all.merge(c_reg[['trial_id_keep', 'additional_ids']].drop_duplicates(),
                              left_on='TrialID',
                              right_on='trial_id_keep',
                              how='left').drop('trial_id_keep', axis=1).rename(columns=
                                                                               {'additional_ids':
                                                                                'cross_registrations'}
                                                                              ).reset_index(drop=True)

df_cond_all['cross_registrations'] = df_cond_all['cross_registrations'].fillna('None')_____no_output_____

Normalisation and data cleaning of all fields. This will be expanded each update as more trials get added and more registries start to add trials with their own idiosyncratic data categories. _____no_output_____

#A small function to help quickly check the contents of various columns

def check_fields(field):
    return df_cond_all[field].unique()

#check_fields('Phase')

#Check fields for new unique values that require normalisation
#for x in check_fields('Countries'):
#    print(x)_____no_output_____#Data cleaning various fields.
#One important thing we have to do is make sure there are no nulls or else the data won't properly load onto the website

#semi-colons in the intervention field mess with CSV
df_cond_all['Intervention'] = df_cond_all['Intervention'].str.replace(';', '')

#Study Type
obv_replace = ['Observational [Patient Registry]', 'observational', 'Observational Study']
int_replace = ['interventional', 'Interventional clinical trial of medicinal product', 'Treatment',
               'INTERVENTIONAL', 'Intervention', 'Interventional Study', 'PMS']
hs_replace = ['Health services reaserch', 'Health Services reaserch', 'Health Services Research']

df_cond_all['Study_type'] = (df_cond_all['Study_type'].str.replace(' study', '')
                             .replace(obv_replace, 'Observational').replace(int_replace, 'Interventional')
                             .replace('Epidemilogical research', 'Epidemiological research')
                             .replace(hs_replace, 'Health services research')
                             .replace('Others,meta-analysis etc', 'Other'))

#phase
df_cond_all['Phase'] = df_cond_all['Phase'].fillna('Not Applicable')
na = ['0', 'Retrospective study', 'Not applicable', 'New Treatment Measure Clinical Study', 'Not selected',
      'Phase 0', 'Diagnostic New Technique Clincal Study', '0 (exploratory trials)', 'Not Specified']
p1 = ['1', 'Early Phase 1', 'I', 'Phase-1', 'Phase I']
p12 = ['1-2', '2020-02-01 00:00:00', 'Phase I/II', 'Phase 1 / Phase 2', 'Phase 1/ Phase 2', '02-Jan',
       'Human pharmacology (Phase I): yes\nTherapeutic exploratory (Phase II): yes\nTherapeutic confirmatory - (Phase III): no\nTherapeutic use (Phase IV): no\n']
p2 = ['2', 'II', 'Phase II', 'IIb', 'Phase-2', 'Phase2',
      'Human pharmacology (Phase I): no\nTherapeutic exploratory (Phase II): yes\nTherapeutic confirmatory - (Phase III): no\nTherapeutic use (Phase IV): no\n']
p23 = ['Phase II/III', '2020-03-02 00:00:00', 'II-III', 'Phase 2 / Phase 3', 'Phase 2/ Phase 3', '2-3', '03-Feb',
       'Human pharmacology (Phase I): no\nTherapeutic exploratory (Phase II): yes\nTherapeutic confirmatory - (Phase III): yes\nTherapeutic use (Phase IV): no\n']
p3 = ['3', 'Phase III', 'Phase-3', 'III',
      'Human pharmacology (Phase I): no\nTherapeutic exploratory (Phase II): no\nTherapeutic confirmatory - (Phase III): yes\nTherapeutic use (Phase IV): no\n']
p34 = ['Phase 3/ Phase 4', 'Phase III/IV',
       'Human pharmacology (Phase I): no\nTherapeutic exploratory (Phase II): no\nTherapeutic confirmatory - (Phase III): yes\nTherapeutic use (Phase IV): yes\n']
p4 = ['4', 'IV', 'Post Marketing Surveillance', 'Phase IV', 'PMS',
      'Human pharmacology (Phase I): no\nTherapeutic exploratory (Phase II): no\nTherapeutic confirmatory - (Phase III): no\nTherapeutic use (Phase IV): yes\n']

df_cond_all['Phase'] = (df_cond_all['Phase'].replace(na, 'Not Applicable').replace(p1, 'Phase 1')
                        .replace(p12, 'Phase 1/Phase 2').replace(p2, 'Phase 2')
                        .replace(p23, 'Phase 2/Phase 3').replace(p3, 'Phase 3').replace(p34, 'Phase 3/Phase 4')
                        .replace(p4, 'Phase 4'))

#Fixing Observational studies incorrectly given a Phase in ICTRP data
m = ((df_cond_all.Phase.str.contains('Phase')) & (df_cond_all.Study_type == 'Observational'))
df_cond_all['Phase'] = df_cond_all.Phase.where(~m, 'Not Applicable')

#Recruitment Status
df_cond_all['Recruitment_Status'] = df_cond_all['Recruitment_Status'].replace('Not recruiting', 'Not Recruiting')
df_cond_all['Recruitment_Status'] = df_cond_all['Recruitment_Status'].fillna('No Status Given')

#Get rid of messy accents
def norm_names(x):
    if isinstance(x,float):
        return x
    else:
        text = unidecode.unidecode(x)
        normed = unicodedata.normalize('NFKD', text).encode('ASCII', 'ignore').decode()
        return normed

df_cond_all['Primary_sponsor'] = df_cond_all.Primary_sponsor.apply(norm_names)
df_cond_all['Primary_sponsor'] = df_cond_all['Primary_sponsor'].replace('NA', 'No Sponsor Name Given')
df_cond_all['Primary_sponsor'] = df_cond_all['Primary_sponsor'].replace('nan', 'No Sponsor Name Given')_____no_output_____#Countries
df_cond_all['Countries'] = df_cond_all['Countries'].fillna('No Country Given').replace('??', 'No Country Given')

china_corr = ['Chian', 'China?', 'Chinese', 'Wuhan', 'Chinaese', 'china', 'Taiwan, Province Of China',
              "The People's Republic of China"]

country_values = df_cond_all['Countries'].tolist()

new_list = []

for c in country_values:
    country_list = []
    if isinstance(c, float):
        country_list.append('No Sponsor Name Given')
    elif c == 'No Sponsor Name Given':
        country_list.append('No Sponsor Name Given')
    elif c in china_corr:
        country_list.append('China')
    elif c in ['Iran (Islamic Republic of)', 'Iran, Islamic Republic of']:
        country_list.append('Iran')
    elif c in ['Viet nam', 'Viet Nam']:
        country_list.append('Vietnam')
    elif c in ['Korea, Republic of', 'Korea, Republic Of', 'KOREA'] :
        country_list.append('South Korea')
    elif c in ['USA', 'United States of America', 'U.S.']:
        country_list.append('United States')
    elif c == 'Japan,Asia(except Japan),Australia,Europe':
        country_list = ['Japan', 'Australia', 'Asia', 'Europe']
    elif c == 'Japan,Asia(except Japan),North America,South America,Australia,Europe,Africa':
        country_list = ['Japan, Asia(except Japan), North America, South America, Australia, Europe, Africa']
    elif c == 'The Netherlands':
        country_list.append('Netherlands')
    elif c == 'England':
        country_list.append('United Kingdom')
    elif c == 'Japan,North America':
        country_list = ['Japan', 'North America']
    elif c == 'Czechia':
        country_list.append('Czech Republic')
    elif c == 'ASIA':
        country_list.append('Asia')
    elif c == 'EUROPE':
        country_list.append('Europe')
    elif c == 'MALAYSIA':
        country_list.append('Malaysia')
    elif c in ['Congo', 'Congo, Democratic Republic', 'Congo, The Democratic Republic of the']:
        country_list.append('Democratic Republic of Congo')
    elif c in ["C√¥te D'Ivoire", 'Cote Divoire', "CÃ´te D'Ivoire"]:
        country_list.append("Cote d'Ivoire")
    elif c in ['Türkiye', 'TÃ¼rkiye', 'TÃƒÂ¼rkiye']:
        country_list.append('Turkey')
    elif c == 'SOUTH AMERICA':
        country_list.append('South America')
    elif c == 'AFRICA':
        country_list.append('Africa')
    elif c == 'italy':
        country_list.append('Italy')
    elif ';' in c:
        c_list = c.split(';')
        unique_values = list(set(c_list))
        for v in unique_values:
            if v in china_corr:
                country_list.append('China')
            elif v in ['Iran (Islamic Republic of)', 'Iran, Islamic Republic of']:
                country_list.append('Iran')
            elif v in ['Korea, Republic of', 'Korea, Republic Of', 'KOREA']:
                country_list.append('South Korea')
            elif v in ['Viet nam', 'Viet Nam']:
                country_list.append('Vietnam')
            elif v in ['USA', 'United States of America']:
                country_list.append('United States')
            elif v == 'The Netherlands':
                country_list.append('Netherlands')
            elif v == 'England':
                country_list.append('United Kingdom')
            elif v == 'Czechia':
                country_list.append('Czech Republic')
            elif v == 'ASIA':
                country_list.append('Asia')
            elif v == 'EUROPE':
                country_list.append('Europe')
            elif v == 'MALAYSIA':
                country_list.append('Malaysia')
            elif v in ['Congo', 'Congo, Democratic Republic', 'Congo, The Democratic Republic of the']:
                country_list.append('Democratic Republic of Congo')
            elif v in ["C√¥te D'Ivoire", 'Cote Divoire', "CÃ´te D'Ivoire"]:
                country_list.append("Cote d'Ivoire")
            elif v in ['Türkiye', 'TÃ¼rkiye', 'TÃƒÂ¼rkiye']:
                country_list.append('Turkey')
            elif v == 'SOUTH AMERICA':
                country_list.append('South America')
            elif v == 'AFRICA':
                country_list.append('Africa')
            elif v == 'italy':
                country_list.append('Italy')
            else:
                country_list.append(v)
    else:
        country_list.append(c.strip())
    new_list.append(', '.join(country_list))

df_cond_all['Countries'] = new_list_____no_output_____

Last space for manual intervention. This will include manual normalisation of new names, any updates to the normalisation schedule from the last update, and updating manually-coded intervention type data._____no_output_____

#Normalizing sponsor names
#Run this cell, updating the spon_norm csv you are loading after manual adjusting
#until you get the 'All sponsor names normalized' to print

spon_norm = pd.read_excel('manual_data.xlsx', sheet_name = 'sponsor')

df_cond_norm = df_cond_all.merge(spon_norm, left_on = 'Primary_sponsor', right_on ='unique_spon_names', how='left')
df_cond_norm = df_cond_norm.drop('unique_spon_names', axis=1)

new_unique_spon_names = (df_cond_norm[df_cond_norm['normed_spon_names'].isna()][['Primary_sponsor', 'TrialID']]
                        .groupby('Primary_sponsor').count())

if len(new_unique_spon_names) > 0:
    new_unique_spon_names.to_csv('to_norm.csv')
    print('Update the normalisation schedule and rerun')
else:
    print('All sponsor names normalized')_____no_output_____#Integrating intervention type data
#Once again, run to bring in the old int-type data, islolate the new ones, update, and rerun until
#producing the all-clear message

int_type = pd.read_excel('manual_data.xlsx', sheet_name = 'intervention')

df_cond_int = df_cond_norm.merge(int_type[['trial_id', 'study_category',
                                           'intervention', 'intervention_list']],
                                 left_on = 'TrialID', right_on = 'trial_id', how='left')

df_cond_int = df_cond_int.drop('trial_id', axis=1)

new_int_trials = df_cond_int[(df_cond_int['study_category'].isna()) | (df_cond_int['intervention'].isna())]

if len(new_int_trials) > 0:
    new_int_trials[['TrialID', 'Public_title', 'Intervention', 'study_category',
                    'intervention', 'intervention_list']].to_csv('int_to_assess.csv')
    print('Update the intervention type assessments and rerun')
else:
    print('All intervention types matched')
    df_cond_int = df_cond_int.drop('Intervention', axis=1).reset_index(drop=True)_____no_output_____#Can use this cell to output counts of values from columns

treatments = df_cond_int[df_cond_int.study_category == 'Drug']['study_category'].tolist()
countries = df_cond_int.Countries.to_list()

def var_counts(var_list, split_char, lower=False):
    final_list = []
    for v in var_list:
        t_list = v.split(split_char)
        for l in t_list:
            if lower:
                final_list.append(l.lower().strip())
            else:
                final_list.append(l.strip())
    return Counter(final_list)_____no_output_____comp_dates = pd.read_excel('manual_data.xlsx', sheet_name = 'Completion Dates')
df_comp_dates = df_cond_int.merge(comp_dates,
                                  left_on='TrialID', right_on='trialid',
                                  how='left', indicator=True).drop('trialid', axis=1)

print('These trials are missing completion date data:')
print(df_comp_dates[df_comp_dates['_merge'] == 'left_only'].TrialID.to_list())

df_comp_dates = df_comp_dates.drop('_merge', axis=1).reset_index(drop=True)

df_comp_dates['primary_completion_date'] = (pd.to_datetime(df_comp_dates['primary_completion_date'],
                                                          errors='coerce',
                                                          format='%Y-%m-%d')
                                            .fillna('Not Available').apply(fix_date))

df_comp_dates['full_completion_date'] = (pd.to_datetime(df_comp_dates['full_completion_date'],
                                                          errors='coerce',
                                                          format='%Y-%m-%d')
                                            .fillna('Not Available').apply(fix_date))_____no_output_____#check for any results on ICTRP

results_checked = ['NCT04323592', 'JPRN-UMIN000040520', 'NCT04410159', 'NCT04422561', 'NCT04491994',
                   'ACTRN12620000869976', 'KCT0005226', 'NCT04280705', 'NCT04343261', 'NCT04523831',
                   'NCT04491240', 'NCT04343092', 'NCT04425850', 'NCT04542694', 'JPRN-UMIN000040405',
                   'JPRN-jRCTs041190120']

ictrp_results = df_comp_dates[(df_comp_dates.has_results.notnull()) | (df_comp_dates.has_results.notnull())]

if len(ictrp_results) > 0:
    print(f'There are {len(ictrp_results) - len(results_checked)} results to check for: {list(set(ictrp_results.TrialID.tolist()) - set(results_checked))}')
else:
    print('There are no results to check')

#If results cross-check with results already collected in 'manual_data' excel file and add any new trial results._____no_output_____results = pd.read_excel('manual_data.xlsx', sheet_name = 'Results')
df_results = df_comp_dates.merge(results,
                                 left_on='TrialID',
                                 right_on='trialid',
                                 how='left').drop('trialid', axis=1)

df_results['results_link'] = df_results['results_link'].fillna('No Results')
df_results['results_type'] = df_results['results_type'].fillna('No Results')

df_results['results_publication_date'] = (pd.to_datetime(df_results['results_publication_date'],
                                                          errors='coerce',
                                                          format='%Y-%m-%d')
                                            .fillna('No Results').apply(fix_date))_____no_output_____#Final organising

col_names = []

for col in list(df_results.columns):
    col_names.append(col.lower())

df_results.columns = col_names

reorder = ['trialid', 'source_register', 'date_registration', 'date_enrollement', 'retrospective_registration',
           'normed_spon_names', 'recruitment_status', 'phase', 'study_type', 'countries', 'public_title',
           'acronym', 'study_category', 'intervention', 'intervention_list', 'target_enrollment',
           'primary_completion_date', 'full_completion_date', 'web_address', 'results_type',
           'results_publication_date', 'results_link', 'last_refreshed_on', 'cross_registrations']

df_final = df_results[reorder].reset_index(drop=True).drop_duplicates().reset_index()
df_final['acronym'] = df_final.acronym.fillna('')
df_final['last_refreshed_on'] = pd.to_datetime(df_final['last_refreshed_on'])_____no_output_____#Checking for any null values
df_final[df_final.isna().any(axis=1)]_____no_output_____#Any fixing of weird null values
#For this time it was just giving a value to EUCTR2020-002234-32-BE
df_final['date_enrollement'] = df_final['date_enrollement'].fillna(pd.to_datetime('2020-11-24'))_____no_output_____#Quick look at the data
df_final.head(10)_____no_output_____#Export final dataset
df_final.to_csv(f'processed_data_sets/trial_list_{this_extract_date}.csv', index=False)

df_final.to_csv(f'tableau_data/current_data.csv', index=False)_____no_output_____just_results = df_final[df_final.results_type != 'No Results']

results_total = len(just_results)

print(f'There are {results_total} trials with results')_____no_output_____#Export json for website
import json
with open("website_data/trials_latest.json", "w") as f:
    json.dump({"data": df_final.astype(str).values.tolist()}, f, indent=2)

with open("website_data/results_latest.json", "w") as f:
    json.dump({"data": just_results.astype(str).values.tolist()}, f, indent=2)    _____no_output_____

# Overall Trend in Registered Trials Graph_____no_output_____

just_reg = df_final[['trialid', 'date_registration']].reset_index(drop=True)
#just_reg = mar18[['trialid', 'date_registration']].reset_index(drop=True)
#just_reg['date_registration'] = pd.to_datetime(just_reg['date_registration'], format='%d/%m/%Y')

#catch old registrations that were expanded to include COVID, we can get rid of these for now
just_reg = just_reg[just_reg['date_registration'] >= pd.Timestamp(2020,1,1)].reset_index(drop=True)_____no_output_____just_reg.index = just_reg['date_registration']

grouped = just_reg.resample('W').count()
cumsum = grouped.cumsum()_____no_output_____import matplotlib.pyplot as plt

labels = []

for x in list(grouped.index):
    labels.append(str(x.date()))

x_pos = [i for i, _ in enumerate(labels)]

#fig, ax = plt.subplots(figsize=(10,5), dpi = 300)

#l1 = plt.plot(x_pos, grouped['trialid'], marker = 'o')
#l2 = plt.plot(x_pos, cumsum['trialid'], marker = 'o')

#for i, j in zip(x_pos[1:], grouped['trialid'].tolist()[1:]):
#    ax.annotate(str(j), (i,j), xytext = (i-.1, j-50))

#for i, j in zip(x_pos, cumsum['trialid']):
#    ax.annotate(str(j), (i,j), xytext = (i-.2, j+25))

gr = grouped['trialid'].to_list()
cs = cumsum['trialid'].to_list()

#plt.xticks(x_pos, labels, rotation=45, fontsize=8)
#plt.ylim(-50, 2500)
#plt.xlabel('Week Ending Date')
#plt.ylabel('Registered Trials')
#plt.title('Registered COVID-19 Trials by Week on the ICTRP')
#plt.legend(('New Trials', 'Cumulative Trials'), loc=2)
#plt.savefig(f'trial_count_{last_extract_date}.png')
#plt.show()_____no_output_____import plotly.graph_objects as go

labels = []

for x in list(grouped.index):
    labels.append(str(x.date()))

fig = go.Figure()

fig.add_trace(go.Scatter(x=labels[:-1], y=grouped['trialid'][:-1], fill=None, name='New Trials'))

fig.add_trace(go.Scatter(x=labels[:-1], y=cumsum['trialid'][:-1], fill=None, name='Cumulative Trials'))

fig.update_layout(title={'text': 'Registered COVID-19 Trials by Week', 'xanchor': 'center', 'x': 0.5},
                  xaxis_title='Week Ending Date',
                  yaxis_title='Registered Trials',
                  legend = dict(x=0, y=1, traceorder='normal', bgcolor='rgba(0,0,0,0)'))

fig.show()
fig.write_html("html_figures/registered trials.html")_____no_output_____int_types = df_final.study_category.value_counts()
int_types_____no_output_____treatment_dict = dict(drugs = int_types['Drug'] + int_types['Drug (Chemoprophylaxis)'],
                      atmp = int_types['ATMP'],
                      clinical_char = (int_types['Clinical Presentation'] + int_types['Diagnostics'] +
                                       int_types['Prognosis'] + int_types['Clinical Presentation (Epidemiology)']),
                      drug_other_combo = (int_types['Drug (+ ATMP + Other (renal))'] + int_types['Drug (+ ATMP)'] +
                                          int_types['ATMP (+ Drug)'] + int_types['Drug (+ Chemoprophylaxis)']),
                      supp = int_types['Supplement'],
                      geno = int_types['Genomics'],
                      th = int_types['Telehealth'],
                      pro = int_types['Procedure'],
                      tm = int_types[[s.startswith('Traditional Medicine') for s in int_types.index]].sum(),
                      other = (int_types[[s.startswith('Other') for s in int_types.index]].sum()
                               + int_types['Health System'])
                     )

fig = go.Figure(go.Bar(
            x=list(treatment_dict.values()),
            y=['Drugs', 'ATMP', 'Clinical Characteristics', 'Multiple Therapies', 'Supplement', 'Genomics',
               'Telehealth', 'Procedure', 'Traditional Medicine', 'Other'],
            orientation='h'))

fig.update_layout(title={'text': 'Intervention Type of Registered Trials', 'xanchor': 'center', 'x': 0.5},
                  xaxis_title='Number of Trials')

fig.show()
fig.write_html('html_figures/int_bar.html')_____no_output_____fig = go.Figure(go.Bar(
            x=df_final.source_register.value_counts().values,
            y=df_final.source_register.value_counts().index,
            orientation='h'))

fig.update_layout(title={'text': 'Registered Studies by Trial Registry', 'xanchor': 'center', 'x': 0.55},
                  xaxis_title='Number of Studies Registered',
                  yaxis=dict(autorange="reversed"))

fig.show()
fig.write_html('html_figures/registries_bar.html')_____no_output_____treatments = df_final[((df_final.study_category.str.contains('Drug')) | (df_final.study_category.str.contains('ATMP'))) & ~(df_final.study_category.str.contains('Traditional Medicine'))]['intervention_list'].tolist()
common_treatments = pd.DataFrame(var_counts(treatments, ';', lower=True).most_common())
common_treatments.columns = ['treatment', 'trial_count']

fig = go.Figure(go.Bar(
            x=common_treatments[common_treatments.trial_count >= 15]['trial_count'],
            y=common_treatments[common_treatments.trial_count >= 15]['treatment'],
            orientation='h'))

fig.update_layout(title={'text': 'Most Commonly Studied Drugs & ATMPs (n>=15)', 'xanchor': 'center', 'x': 0.5},
                  xaxis_title='Number of Studies Registered',
                  yaxis=dict(autorange="reversed", dtick=1))

fig.show()
fig.write_html('html_figures/treatment_bar.html')_____no_output_____countries = df_final.countries.to_list()
most_studies = pd.DataFrame(var_counts(countries, ',', lower=False).most_common())
most_studies.columns = ['country', 'trial_count']

fig = go.Figure(go.Bar(
            x=most_studies[(most_studies.trial_count >= 50) & (most_studies.country != 'No Country Given')]['trial_count'],
            y=most_studies[(most_studies.trial_count >= 50) & (most_studies.country != 'No Country Given')]['country'],
            orientation='h'))

fig.update_layout(title={'text': 'Most Common Study Locations (n>=50)', 'xanchor': 'center', 'x': 0.5},
                  xaxis_title='Number of Studies Registered',
                  yaxis=dict(autorange="reversed", dtick=1))

fig.show()
fig.write_html('html_figures/location_bar.html')_____no_output_____


# Notebook from LDSSA/batch4-students
Path: S04 - Text Classification/BLU08 - Dimensionality Reduction/BLU08 - Learning Notebook - Part 1 of 3 - Dimensionality Reduction and Feature Selection.ipynb

## Part 1 - Dimensionality Reduction and Feature Selection

Thinking in high dimensions is particularly hard. A 1-D dot would hardly be able to imagine a 2D world. In the same way, the square below struggles to think of our 3D world. For us, it is very difficult to imagine dimensions above the 3rd. We have proxies for thinking of a 4th dimension - passing of time, a 3D surface with some extra measure represented by color, etc - but this starts to become really difficult to work above the 4th dimension.

![dimensionality](./media/flatland.png)

Thankfully, when dealing with ML problems, you can go above three features without trying to imagine how they would look like (Lucky you!). However, you still need to understand how the number of dimensions might affect - either positively or negatively - your models and how to work with high dimensional spaces. Don't worry, we will guide you through it._____no_output_____

import string
import re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from pylab import barh,plot,yticks,show,grid,xlabel,figure,cla,close
from nltk.tokenize import WordPunctTokenizer

from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn import svm
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import SelectFromModel
from sklearn.feature_selection import SelectKBest, chi2/Users/catarinasilva/.pyenv/versions/3.8.5/envs/batch4-blu8/lib/python3.8/site-packages/pandas/compat/__init__.py:120: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.
  warnings.warn(msg)

## 1. The curse of dimensionality

So far you learned how to handle text data by transforming it into a vectorized feature space. Namely, we mostly used preprocessing tricks and some sort of count over words - remember the CountVectorizer and TF-IDF - to generate our feature space. However, as you might have realized, the number of features for these problems is huge, in particular if you want to cover all language. In the limit, your feature space will cover the entire vocabulary!

The effect of high dimensionality features when modelling these problems is called the **curse of dimensionality**. You can probably already see some of the effects of these curse, right?

* To start with, more features, will obviously mean a longer training process
* On another hand, a higher dimensionality can actually hurt the classifier performance

#### But I thought "more features" meant "more accuracy"...

This is not always true. When you have high dimensionality problems, the use of all the features might actually hurt your model. The model becomes more prone to overfitting and might have worse accuracy in points outside your training data._____no_output_____
## 2.  Basic Feature Selection

One of the ways you might think of to reduce your feature dimensionality is by performing feature selection. We are going to walk you through some methods with an actual example. Let's start by loading some data - we are going to use the twitter dataset of republican and democrat tweets_____no_output_____### 2.1 - Get your dataset

Start by importing the dataset. _____no_output_____

df = pd.read_csv('./datasets/twitter_rep_dem_data_small.csv')_____no_output_____

### 2.2 - Get to know our problem

We'll first learn our categories and see a few examples of how our training data looks like. _____no_output_____

print('Categories:')
print(', '.join(set(df.Party)))Categories:
Republican, Democrat

Check class balances_____no_output_____

df.Party.value_counts()_____no_output_____

Pretty balanced!_____no_output_____

df.head(10)_____no_output_____

You'll notice that our tweets are just text with some particularities. For example, it is common to have twitter handles in the text, defined by the "@" character. Our data also has three columns, but we are going to ignore the *Handle* column for now and just focus on classifying *Tweets* with *Party* labels._____no_output_____### 2.3 - Feature Extraction

Our data is simply raw text, each element a document to be classified, with a corresponding label, which is the Party of the tweet. Pretty simple, right?

Since you are a great student, you went thoroughly through BLU07 and you already know how to prepare, handle text and extract some simple features. So let's process our data and use TfidfVectorizer with a range of 1-2 ngrams to get us some simple features.

First let's apply simple tokenization. To avoid overfitting to twitter related information, like the handles tagged in the messages, let's remove those, so our focus is only on the language._____no_output_____

handle_removal = lambda doc: re.subn(r'@\w+','', doc.lower())[0]
df['Tweet'] = df['Tweet'].map(handle_removal)

simple_tokenizer = lambda doc: " ".join(WordPunctTokenizer().tokenize(doc))
df['Tweet'] = df['Tweet'].map(simple_tokenizer)

df.head(10)_____no_output_____

Now let's split our data and apply some vectorization. Let's pick a random seed so the results are replicable._____no_output_____

seed = 42_____no_output_____

_____no_output_____

train_data, test_data = train_test_split(df, test_size=0.3, random_state=seed)

print('Training examples: {}'.format(train_data.size))
print('Test examples: {}\n'.format(test_data.size))

# important: only train the vectorizer on the training data
vectorizer = TfidfVectorizer(ngram_range=(1,2))
%timeit vectorizer.fit(train_data.Tweet)

X_train = vectorizer.transform(train_data.Tweet)
X_test = vectorizer.transform(test_data.Tweet)

y_train = train_data.Party
y_test = test_data.PartyTraining examples: 38889
Test examples: 16668

2.9 s ± 409 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

### 2.4 - Getting our baseline

Let's get our baseline accuracy and measure the time it takes to fit a Naive Bayes Model, a model you should be familiar with and which in NLP comes hand in hand with the Bag Of Words representation (if you don't get the joke below, eventually you should go read about naive bayes).

_____no_output_____

clf =  MultinomialNB()
%timeit clf.fit(X_train, y_train)

y_pred = clf.predict(X_test)
print('Accuracy: {}'.format(accuracy_score(y_pred, y_test)))104 ms ± 27.1 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7595392368610511

### 2.5 - Feature selection

Now that we have our baseline, let's start by looking into the number of features used in our classifier:_____no_output_____

X_train.shape_____no_output_____

So far so good, and our classifier even trained pretty fast, but a 140K-dimensional space is obviously very difficult to interpret (think back to the fact that even 4D is difficult for us to grasp). Let's instead try to extract our K most important words. One way that you might think to do this is actually to just get the features corresponding to the most frequent terms. In fact, our TfidfVectorizer already has an option for that. Let's see the impact on our training speed and accuracy._____no_output_____

for k in [10, 100, 1000, 5000, 10000, 50000, 100000]:
    print('Using {} features'.format(k))
    print('----'.format(k))

    X_train = train_data.Tweet
    X_test = test_data.Tweet

    vectorizer_truncated = TfidfVectorizer(ngram_range=(1,2), max_features=k)
    vectorizer_truncated.fit(X_train)

    X_train_truncated = vectorizer_truncated.transform(X_train)
    X_test_truncated = vectorizer_truncated.transform(X_test)

    clf =  MultinomialNB()
    %timeit clf.fit(X_train_truncated, y_train)
    y_pred = clf.predict(X_test_truncated)

    print('Accuracy: {}\n'.format(accuracy_score(y_pred, y_test)))Using 10 features
----
93.7 ms ± 18.7 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.5259179265658748

Using 100 features
----
64.4 ms ± 7.28 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.5815334773218143

Using 1000 features
----
57.5 ms ± 5.79 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.6637868970482361

Using 5000 features
----
65.4 ms ± 5.77 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7177825773938085

Using 10000 features
----
54 ms ± 11.5 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7341612670986322

Using 50000 features
----
57.2 ms ± 3.39 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7543196544276458

Using 100000 features
----
96.6 ms ± 11.3 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7543196544276458

Ok, so no amazing effects. Let's look into the actual top K-features to see if they make sense._____no_output_____

K=10
vectorizer_truncated = TfidfVectorizer(ngram_range=(1,2), max_features=K)
vectorizer_truncated.fit(X_train)
feature_names = vectorizer_truncated.get_feature_names()
for f in feature_names:
    print(f)and
co
for
https
https co
in
of
rt
the
to

As you can see, the top 10 features are basically meaningless when thinking of our classes. In the next cell you will see that the counts of these words are balanced between classes._____no_output_____

sub_punct = lambda s: re.sub(r'\s+\W*[a-z]?\W*\s+', ' ', s)_____no_output_____for feature in feature_names:
    print('Documents that contain the word "{}"'.format(feature))
    print('----')
    docs = X_train.apply(sub_punct).str.lower().str.contains(feature)
    print(str(y_train[docs].value_counts()) + '\n\n')Documents that contain the word "and"
----
Democrat      2410
Republican    2334
Name: Party, dtype: int64

Documents that contain the word "co"
----
Democrat      5571
Republican    5493
Name: Party, dtype: int64

Documents that contain the word "for"
----
Republican    2307
Democrat      2188
Name: Party, dtype: int64

Documents that contain the word "https"
----
Democrat      4818
Republican    4702
Name: Party, dtype: int64

Documents that contain the word "https co"
----
Democrat      4775
Republican    4616
Name: Party, dtype: int64

Documents that contain the word "in"
----
Republican    5103
Democrat      5101
Name: Party, dtype: int64

Documents that contain the word "of"
----
Democrat      2294
Republican    2235
Name: Party, dtype: int64

Documents that contain the word "rt"
----
Republican    2755
Democrat      2670
Name: Party, dtype: int64

Documents that contain the word "the"
----
Democrat      4191
Republican    4139
Name: Party, dtype: int64

Documents that contain the word "to"
----
Democrat      4287
Republican    4182
Name: Party, dtype: int64

Tip: Notice that most of these words are normally considered stopwords. Try to exclude stopwords from the TfidfVectorizer and see what new top features you obtain. You will probably see that there will be some common words and some "political discourse" words, but not really democrat/republican specific.

But let's move on to more meaningful approaches._____no_output_____## 3.  Feature Selection through statistical analysis

Basic feature selection methods might actually work sometimes, in particular if you pick a reasonable heuristic to decide on which features to choose. In our case, obviously, just picking the higher counts is not that good, since it does not provide any useful information on the labels. But you could imagine for example trying to pick as features words that appear only (or almost only) on one of our classes.

Although this might seem a good idea, depending on your problem, you would probably not want to lose that much time thinking about heuristics, implementing them and comparing them. This is where statistical tests are useful. You don't have to reason on the features you are using, these tests use your data to provide insights on your features.

##### Chi-squared test

The chi-squared test is one of these tests. The chi-squared formula measures how much expected counts and observed counts of variables/distributions deviate from each other. It can be used to test for independce between two variables, like defined by the equation below, where $O_{x_1x_2}$ is the observation of the conjuction of variables and $E_{x_1x_2}$ the corresponding expected value, this is, the expected value given our hypothesis *$H_0$: the variables are independent*.

$$\chi^2 = \sum{\frac{(O_{x_1x_2} - E_{x_1x_2})^2}{E_{x_1x_2}}}$$

For feature selection we want to test the independence of our features from the class labels. In our particular case, we define $x_1=t$ as our term or word and $x_2=c$ as our class label. A small chi-squared value will mean that the term is closer to independence from the class, and a big value that it is very dependent on the class.

Knowing the details of the chi-squared can be useful for you, but in the context of our BLU is not our primary goal and there are more useful methods for text features that you will learn about in the following notebooks. However, we provide at the end of this BLU a more detailed explanation of this test, and some examples, if you wish to understand it better (see Annex A).

We will lean on the previous example and show you how chi-squared would help us select more meaningful features._____no_output_____## 3.1 - Setup problem

Like before, let's fetch our data, extract its features, run a baseline, and move from there._____no_output_____

stat_df = pd.read_csv('./datasets/twitter_rep_dem_data_small.csv')

stat_df['Tweet'] = stat_df['Tweet'].map(handle_removal)
stat_df['Tweet'] = stat_df['Tweet'].map(simple_tokenizer)

stat_train_data, stat_test_data = train_test_split(stat_df, test_size=0.3, random_state=seed)

stat_vectorizer = TfidfVectorizer(ngram_range=(1,2))
%timeit stat_vectorizer.fit(stat_train_data.Tweet)

stat_X_train = stat_vectorizer.transform(stat_train_data.Tweet)
stat_X_test = stat_vectorizer.transform(stat_test_data.Tweet)

stat_y_train = stat_train_data.Party
stat_y_test = stat_test_data.Party

stat_clf =  MultinomialNB()
%timeit stat_clf.fit(stat_X_train, stat_y_train)

stat_pred = stat_clf.predict(stat_X_test)

print('Accuracy: {}'.format(accuracy_score(stat_pred, stat_y_test)))1.98 s ± 365 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
99.4 ms ± 8.18 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7595392368610511

## 3.2 - Feature Selection from chi-squared

We will now use the chi2 and obtain some chi-squared values for our features. _____no_output_____

chi_values, p_values = chi2(stat_X_train, stat_y_train)_____no_output_____

We can plot the most dependent features from the chi-squared values._____no_output_____

feature_names = stat_vectorizer.get_feature_names()

cla()   # Clear axis
close() # Close a figure window

figure(figsize=(12,10))
zipped_chi_squared = zip(feature_names, chi_values)
sorted_chi_values = sorted(zipped_chi_squared, key=lambda x:x[1])
top_chi_values = list(zip(*sorted_chi_values[-30:]))

x = range(len(top_chi_values[1]))
labels = top_chi_values[0]
barh(x, list(top_chi_values)[1], align='center', alpha=.2, color='g')
yticks(x, labels)
xlabel('$\chi^2$')
show()_____no_output_____

Actually, scikit-learn already provides a function to directly select the K-best features for our model, so we are going to use that to extract our most important features. We can confirm that the top features selected match the ones with the highest chi-values_____no_output_____

ch2 = SelectKBest(chi2, k=10)
ch2.fit(stat_X_train, stat_y_train)

most_important_features = [feature_names[i] for i in ch2.get_support(indices=True)]
for f in most_important_features:
    print(f)chairman
code
hearing
reform
tax
tax reform
taxcutsandjobsact
taxreform
texas
the taxcutsandjobsact

Now we're getting somewhere, these new features are starting to make sense. We can look into their distribution of  over our documents to get a sense of the relation they share with the labels of the dataset._____no_output_____

for feature in most_important_features:
    print('Documents that contain the word "{}"'.format(feature))
    print('----')
    docs = stat_train_data['Tweet'].apply(sub_punct).str.contains(feature)
    print(str(stat_y_train[docs].value_counts()) + '\n\n')Documents that contain the word "chairman"
----
Republican    196
Democrat        8
Name: Party, dtype: int64

Documents that contain the word "code"
----
Republican    107
Democrat        8
Name: Party, dtype: int64

Documents that contain the word "hearing"
----
Republican    264
Democrat       62
Name: Party, dtype: int64

Documents that contain the word "reform"
----
Republican    426
Democrat       34
Name: Party, dtype: int64

Documents that contain the word "tax"
----
Republican    800
Democrat      217
Name: Party, dtype: int64

Documents that contain the word "tax reform"
----
Republican    126
Democrat        6
Name: Party, dtype: int64

Documents that contain the word "taxcutsandjobsact"
----
Republican    102
Name: Party, dtype: int64

Documents that contain the word "taxreform"
----
Republican    227
Democrat        2
Name: Party, dtype: int64

Documents that contain the word "texas"
----
Republican    121
Democrat        1
Name: Party, dtype: int64

Documents that contain the word "the taxcutsandjobsact"
----
Republican    86
Name: Party, dtype: int64

As you can see, some words are much more common in Republican tweets compared to Democrat tweets. These features are thus much more interpretable and might have a better impact on training._____no_output_____

for k in [10, 100, 1000, 5000, 10000, 50000, 100000, 'all']:
    print('Using {} features'.format(k))
    print('----'.format(k))

    ch2_train = SelectKBest(chi2, k=k)
    ch2_train.fit(stat_X_train, stat_y_train)
    X_train_chi = ch2_train.transform(stat_X_train)
    X_test_chi = ch2_train.transform(stat_X_test)

    clf = MultinomialNB()
    %timeit clf.fit(X_train_chi, stat_y_train)

    y_pred = clf.predict(X_test_chi)

    print('Accuracy: {}\n'.format(accuracy_score(y_pred, stat_y_test)))

Using 10 features
----
58.5 ms ± 6.18 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.4744420446364291

Using 100 features
----
72.9 ms ± 11.5 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.5984521238300936

Using 1000 features
----
54.4 ms ± 5.73 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.709143268538517

Using 5000 features
----
60.4 ms ± 9.85 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7321814254859611

Using 10000 features
----
52.9 ms ± 3.71 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7368610511159107

Using 50000 features
----
82.8 ms ± 24 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7496400287976962

Using 100000 features
----
68.4 ms ± 6.99 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7566594672426206

Using all features
----
82.2 ms ± 9.56 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
Accuracy: 0.7595392368610511

Using 100000 features we get only slightly better results. However, looking at the features themselves, we can clearly see this is a better feature selection method than the previous one, but we could still use a bit more gain and speed improvements.

One thing that feature selection does not take into consideration is feature interaction, which can limit a bit the gains in performance. In the remaining of this BLU you will learn about more elaborate methods to perform dimensionality reduction, in particular some very recent methods that are the standard in text-related tasks. _____no_output_____## 4. Final remarks

After reading this notebook you should understand:

 - What is the curse of dimensionality
 - How can you perform simple feature selection by reasoning about your problem
 - How to apply statistical methods for feature selection by finding dependencies between features and labels

Keep in mind that predicting in the real world is much less theoretical. The performance of these methods will depend a lot on your problem, the size of your dataset, your model choice, your preprocessing. You can use feature selection to improve speed, avoid overfitting, or even just to interpret your features and how they interact with your classes.

-----

**Suggestion**: try to vary the following options/parameters and analyze its impact:

- Experiment a bit more with your preprocessing and see its impact
- Try other feature extraction options such as the simple CountVectorizer
- Use smaller slices of the dataset to see how the dataset size impacts both baseline and feature selected results
- Experiment with other classifiers and the impact of the dimensionality reduction on them
- Try to use the model to evaluate other text data, search for republican/democrat posts/news/blogs and see how well your model classifies each, for example political speeches.

-----

Although we focused on simple heuristics and in the chi-squared method, there are other features selection methods that you might find useful. Some examples of these are:

- Feature selection through variance (sklearn.feature_selection.VarianceThreshold)
- Feature selection through mutual information
- Recursive feature elimination
- Tree-based feature selection

**And remember, these methods just tell us that there is a relation between labels and features, but not the nature of that relation.** Now go and apply these methods!

![correlation](./media/correlation.png)

_____no_output_____-------------

## Annex A.  Details on chi-squared

Let's do a really quick example for you to understand how this works. Let's say we are modeling how characteristics from star trek characters that appear in an episode are related to their death, and among our features we have one particular called "has red t-shirt" which can take only two categorical values: Yes/No.

Let's build a table representing this scenario:

|  Has red t-shirt    | Dies   | Does not die | Total |
|----------------------|--------|--------------|-------|
|         Yes          |   63   |      9       |  72   |
|         No           |   13   |     40       |  53   |
|         total        |   76   |     49       |  125  |

This is what we call a contigency table, and it contains our observed values. Testing for independence of the variables, our expected value is computed from the probabilities $N*P(x_1x_2) = N*P(x_1)P(x_2) $, and we get:

|  Has red t-shirt    | Dies   | Does not die |
|----------------------|--------|--------------|
|         Yes          |  43.78 $$(125 * \frac{76}{125}\frac{72}{125})$$ | 28.22 $$(125 * \frac{49}{125}\frac{72}{125})$$ |
|         No           |  32.22 $$(125 * \frac{76}{125}\frac{53}{125})$$ | 20.78 $$(125 * \frac{49}{125}\frac{53}{125})$$ |

Now we can use these expected values to compute chi-squared using the formula $\chi^2 = \sum{\frac{(O_{x_1x_2} - E_{x_1x_2})^2}{E_{x_1x_2}}}$

|  Has red t-shirt    | Dies   | Does not die |
|----------------------|--------|--------------|
|         Yes          |  8.44 $$(\frac{(63-43.78)^2}{43.78})$$ | 13.09 $$(\frac{(9-28.22)^2}{28.22})$$ |
|         No           |  11.47 $$(\frac{(13-32.22)^2}{32.22})$$ | 17.79 $$(\frac{(40-20.78)^2}{20.78})$$ |

Summing these values, we get a chi-squared of 50.79, which corresponds to a p-value < .00001, which means we are dealing with dependent variables.

_____no_output_____### Chi-squared in BoW context

But how about our context? Our variables are not categorical, so how do we compute this? Actually the chi-squared value can be extended to frequencies, building a contingency table from the feature values and class label. Starting from a table of word frequencies:

|                      |            $C_0$              | ... |             $C_j $               | ... |                         Total               |
|----------------------|---------------------|-----|-------------------------|-----|----------------------------|
|   Word 0             |  $$C_{t_0, c_0}$$   | ... |  $$C_{t_0, c_1}$$       | ... | $$ \sum_j{C_{t_0, c_j}} $$ |
|   Word 1             |  $$C_{t_1, c_0}$$   | ... |  $$C_{t_1, c_1}$$       | ... | $$ \sum_j{C_{t_1, c_j}} $$ |
|    ...               |   ...               | ... |     ...                 | ... |        ...                 |
|   Word i             |  $$C_{t_i, c_0}$$   | ... |  $$C_{t_i, c_1}$$       | ... | $$ \sum_j{C_{t_i, c_j}} $$ |
|    ...               |   ...               | ... |     ...                 | ... |        ...                 |
|   Total       | $$ \sum_i{C_{t_i, c_0}} $$ | ... |$$\sum_i{C_{t_i, c_j}} $$| ... | $$ N = \sum_{i,j}{C_{t_i, c_j}} $$|

We can take each feature ($t=x_i$) and class ($c=c_j$) and assume a table of the form:

|                      | Class j   | Not Class j |  Total |
|----------------------|-----------|-------------|--------|
|   Word i             |  $C_{tc}$ |   $C_{tx}$  |  $C_{tc}$ + $C_{tx}$ |
|   Not Word i         |  $C_{xc}$ |   $C_{xx}$  |  $C_{xc}$ + $C_{xx}$ |
|   Total      |  $C_{tc}$ + $C_{xc}$ |   $C_{tx}$ + $C_{xx}$  |  N = $C_{tc}$ + $C_{xc}$ + $C_{tx}$ + $C_{xx}$ |

Where:

- $C_{tc}$ : counts of co-ocurrences of the term and class
- $C_{tx}$ : counts of ocurrences of the term but not the class
- $C_{xc}$ : counts of ocurrences of the class but not the term
- $C_{xx}$ : counts of ocurrences outside the class and without the term

Notice that you can compute your negative word counts ("Not Word i") by using the totals:

$$C_{xc} = \sum_i{C_{t_i, c}} - C_{tc}  \quad\quad C_{tx} = \sum_j{C_{t, c_j}} - C_{tc} \quad\quad C_{xx} = N - C_{tc} - C_{tx} - C_{xc} $$

The expression can be expanded to the following, for each term $t$ and class $c$:

$$\chi^2(t, c) = \frac{N(C_{tc}C_{xx}-C_{tx}C_{xc})^2}{(C_{tc}+C_{xc})(C_{tx}+C_{xx})(C_{tc}+C_{tx})(C_{xc}+C_{xx})}$$

-----

**Sugestion**: If, like me, you can't move forward without understanding the origin of these expressions, try to get from the initial expression to this expanded form. However, if you get stuck on the math, just go into the notebook **Learning Notebook - Optional - Chi-squared math** and follow the demonstration.

-----

And we will get for all our terms and classes the chi-squared values indication correlation

|                      | Republican ($C_0$)  | Democrat ($C_1$) |
|----------------------|-----------|-------------|
|   Word 0             |  $$\chi^2(t_0, c_0)$$ |  $$\chi^2(t_0, c_1)$$ |
|   Word 1             |  $$\chi^2(t_1, c_0)$$ |  $$\chi^2(t_1, c_1)$$ |
|    ...               |   ...     |     ...     |
|   Word i             |  $$\chi^2(t_i, c_0)$$ |  $$\chi^2(t_i, c_1)$$ |
|    ...               |   ...     |     ...     |

### Chi-squared and TF-IDF

You've seen how to apply chi-squared for categorical values and now for frequencies, more specifically for word frequencies. But we were applying it to TF-IDF values, values that seem to violate the chi-squared rules. The reason why we can apply the chi-squared to TF-IDF values is actually because these are just weighted/scaled frequencies and the probabilities and totals should add up.

### Implementation, finally!

Let's apply this and write a function that receives a matrix with term counts for each label._____no_output_____

def chi_squared(counts):
    """
    Non vectorized version of chi squared function - the idea is that you see the relation with the formula above,
    but you should never use such an inefficient version when actually performing a chi-squared analysis
    """
    print("Applying chi-squared to {} feature and {} classes".format(counts.shape[0], counts.shape[1]))
    chi_values = np.zeros(counts.shape)
    for i in range(counts.shape[0]):
        for j in range(counts.shape[1]):
            n = counts.sum()
            c_tc = counts[i,j]
            c_tx = counts.sum(axis=1)[i,0]-c_tc
            c_xc = counts.sum(axis=0)[0,j]-c_tc
            c_xx = n-c_tc-c_tx-c_xc
            chi_values[i,j] = n*(((c_tc*c_xx)-(c_tx*c_xc))**2)/((c_tc+c_xc)*(c_tx+c_xx)*(c_tc+c_tx)*(c_xc+c_xx))
    return chi_values

def chi_squared_vect(counts):
    """
    Vectorized version of chi squared function - this is still a non-optimized version, but it should run faster than
    the previous function
    """
    print("Applying chi-squared to {} feature and {} classes".format(counts.shape[0], counts.shape[1]))
    n = counts.sum()
    c_tc = counts
    c_tx = counts.sum(axis=1)-counts
    c_xc = counts.sum(axis=0)-counts
    c_xx = n * np.ones(counts.shape) - counts - c_tx - c_xc
    num = n * np.square(np.multiply(c_tc, c_xx)-np.multiply(c_tx, c_xc))
    den = np.multiply(np.multiply(np.multiply(c_tc+c_xc, c_tx+c_xx), c_tc+c_tx), c_xc+c_xx)
    chi_values = np.divide(num, den)
    return chi_values_____no_output_____

### Applying to our previous example

Now we'll apply our functions to a small portion of our data (first 100 tweets) since our implementation is not optimized, in particular the non-vectorized example. If we find the features with higher chi-values, we find our more important features, which are the ones with less independence._____no_output_____

small_vectorizer = TfidfVectorizer(ngram_range=(1,2))
small_vectorizer.fit(train_data.Tweet)
small_X_train = small_vectorizer.transform(train_data.Tweet)
small_y_train = train_data.Party

idx_rep = np.where(small_y_train=='Republican')
idx_dem = np.where(small_y_train=='Democrat')

counts_rep = small_X_train[idx_rep[0], :].sum(axis=0)
counts_dem = small_X_train[idx_dem[0], :].sum(axis=0)
counts = np.concatenate((counts_rep, counts_dem))

# chi_values = chi_squared(counts.transpose())
chi_values_vect = chi_squared_vect(counts.transpose())

feature_names = small_vectorizer.get_feature_names()

best_features = chi_values_vect.argsort(axis=0).tolist()

print("Most important features:\n")
for idx in sorted(best_features[-10:]):
    print(u"{}, value: {}".format(feature_names[idx[0]], chi_values_vect[idx[0], 0]))
Applying chi-squared to 138409 feature and 2 classes
Most important features:

chairman, value: 24.846272877542923
code, value: 12.933225558706729
hearing, value: 14.19673500702519
reform, value: 14.104207101906178
tax, value: 22.33857645946327
tax reform, value: 14.347639988872228
taxcutsandjobsact, value: 14.522494937205447
taxreform, value: 26.959234731744832
texas, value: 15.279411340041444
the taxcutsandjobsact, value: 12.734023819258736

Awesome, we got to the same results as the scikit-learn! Time to move on to the next notebook!_____no_output_____


# Notebook from mukamel-lab/ALLCools
Path: docs/allcools/cell_level/step_by_step/100kb/07-FinalEmbeddingWithCEF.ipynb

# Final Embedding

## Purpose
The purpose of this step is to recalculate the embedding coordinates using consensus clustering labels and the Cluster Enriched Features (CEF) of each cluster.

## Input
- HVF adata file.
- Consensus cluster labels

## Output
- HVF adata file with updated embedding coordinates._____no_output_____## Import_____no_output_____

import seaborn as sns
import matplotlib.pyplot as plt
import anndata
import scanpy as sc
import numpy as np
import pandas as pd
from ALLCools.clustering import cluster_enriched_features, significant_pc_test, log_scale, balanced_pca, tsne
from ALLCools.plot import categorical_scatter_____no_output_____sns.set_context(context='notebook', font_scale=1.3)_____no_output_____

## Parameters_____no_output_____

cell_metadata_path = 'CellMetadata.PassQC.csv.gz'
cluster_adata_path = 'adata.with_coords.h5ad'
ch_adata_path = 'mCH.HVF.h5ad'
cg_adata_path = 'mCG.HVF.h5ad'

# Cluster Enriched Features analysis
top_n=200
alpha=0.05
stat_plot=True

# you may provide a pre calculated cluster version.
# If None, will perform basic clustering using parameters below.
cluster_col = 'L1'
p_cutoff = 0.1_____no_output_____

## Load Data_____no_output_____

adata = anndata.read_h5ad(cluster_adata_path)
ch_adata = anndata.read_h5ad(ch_adata_path)
cg_adata = anndata.read_h5ad(cg_adata_path)_____no_output_____

## Cluster Enriched Features (CEF)

### mCH_____no_output_____

ch_adata.obs[cluster_col] = adata.obs[cluster_col]
cluster_enriched_features(ch_adata,
                          cluster_col=cluster_col,
                          top_n=top_n,
                          alpha=alpha,
                          stat_plot=True)Found 28 clusters to compute feature enrichment score
Computing enrichment score
Computing enrichment score FDR-corrected P values
Selected 3238 unique features

### mCG_____no_output_____

cg_adata.obs[cluster_col] = adata.obs[cluster_col]
cluster_enriched_features(cg_adata,
                          cluster_col=cluster_col,
                          top_n=top_n,
                          alpha=alpha,
                          stat_plot=True)Found 28 clusters to compute feature enrichment score
Computing enrichment score
Computing enrichment score FDR-corrected P values
Selected 2762 unique features

## Save AnnData_____no_output_____

# save adata
ch_adata.write_h5ad(ch_adata_path)
cg_adata.write_h5ad(cg_adata_path)

# save CEF ids
with open(f'{cluster_col}_enriched_features.mCH.txt', 'w') as f:
    features = ch_adata.var_names[ch_adata.var[f'{cluster_col}_enriched_features']]
    f.write('\n'.join(features))
with open(f'{cluster_col}_enriched_features.mCG.txt', 'w') as f:
    features = cg_adata.var_names[cg_adata.var[f'{cluster_col}_enriched_features']]
    f.write('\n'.join(features))_____no_output_____

## PCA

### mCH_____no_output_____

# using Cluster Enriched Features
ch_adata = ch_adata[:, ch_adata.var[f'{cluster_col}_enriched_features']].copy()
# scale
log_scale(ch_adata)
# PCA
balanced_pca(ch_adata, groups=cluster_col)
# Select significant PCs
ch_n_components = significant_pc_test(ch_adata, p_cutoff=p_cutoff)125 components passed P cutoff of 0.1.
Changing adata.obsm['X_pca'] from shape (16985, 200) to (16985, 125)

### mCG_____no_output_____

# using Cluster Enriched Features
cg_adata = cg_adata[:, cg_adata.var[f'{cluster_col}_enriched_features']].copy()
# scale
log_scale(cg_adata)
# PCA
balanced_pca(cg_adata, groups=cluster_col)
# Select significant PCs
cg_n_components = significant_pc_test(cg_adata, p_cutoff=p_cutoff)49 components passed P cutoff of 0.1.
Changing adata.obsm['X_pca'] from shape (16985, 200) to (16985, 49)

### Concatenate_____no_output_____

ch_pcs = ch_adata.obsm['X_pca'][:, :ch_n_components]
cg_pcs = cg_adata.obsm['X_pca'][:, :cg_n_components]

# scale the PCs so CH and CG PCs has the same total var
cg_pcs = cg_pcs / cg_pcs.std()
ch_pcs = ch_pcs / ch_pcs.std()

# total_pcs
total_pcs = np.hstack([ch_pcs, cg_pcs])
adata.obsm['X_pca'] = total_pcs_____no_output_____

## Manifold Learning_____no_output_____

def dump_embedding(adata, name, n_dim=2):
    # put manifold coordinates into adata.obs
    for i in range(n_dim):
        adata.obs[f'{name}_{i}'] = adata.obsm[f'X_{name}'][:, i]
    return adata_____no_output_____

### tSNE_____no_output_____

tsne(adata,
     obsm='X_pca',
     metric='euclidean',
     exaggeration=-1,  # auto determined
     perplexity=30,
     n_jobs=-1)
dump_embedding(adata, 'tsne')_____no_output_____fig, ax = plt.subplots(figsize=(4, 4), dpi=250)
_ = categorical_scatter(data=adata.obs,
                        ax=ax,
                        coord_base='tsne',
                        hue=cluster_col,
                        show_legend=True,
                        text_anno=cluster_col,
                        s=2,
                        max_points=None)_____no_output_____

### UMAP_____no_output_____

sc.pp.neighbors(adata)
try:
    sc.tl.paga(adata, groups=cluster_col)
    sc.pl.paga(adata, plot=False)
    sc.tl.umap(adata, init_pos='paga')
except:
    sc.tl.umap(adata)
dump_embedding(adata, 'umap')_____no_output_____fig, ax = plt.subplots(figsize=(4, 4), dpi=250)
_ = categorical_scatter(data=adata.obs,
                        ax=ax,
                        coord_base='umap',
                        hue=cluster_col,
                        show_legend=True,
                        text_anno=cluster_col,
                        s=2,
                        max_points=None)_____no_output_____

## Save_____no_output_____

adata.write_h5ad('adata.with_coords.h5ad')
adata_____no_output_____# save final clustering result table
cell_meta = pd.read_csv(cell_metadata_path, index_col=0)
final_result = pd.concat([cell_meta, adata.obs], axis=1, sort=True)
final_result.to_csv(f'{cluster_col}.ClusteringResults.csv.gz')
final_result.head()_____no_output_____


# Notebook from NicoSerranoP/PySyft
Path: examples/tutorials/translations/japanese/Part 08 bis - Introduction to Protocols.ipynb

# Part 8 bis - プロトコル入門

### コンテクスト

前回のチュートリアルではPlanについてご紹介しました。今回はProtocolという概念についてです。Protocolは複数のPlanを組み合わせ、リモートワーカーの環境下にデプロイすることができます。デプロイ後は一回のコミュニケーションで実行することができます。

Protocolは複雑な演算処理を複数のワーカーに分散させることができる外観的なオブジェクトです。Protocolの重要な特徴は、ワーカー間で送られたり、検索されたりして、最終的に特定のワーカーにデプロイできる事です。そのため、ユーザーはProtocolをデザインすることで、ワーカーはそれをダウンロード、適用して、プログラムを実行する、なんて事が実現できるのです。

では、見てみましょう。

Authors:
- Théo Ryffel - Twitter [@theoryffel](https://twitter.com/theoryffel) - GitHub: [@LaRiffle](https://github.com/LaRiffle)_____no_output_____### 1. 作成とデプロイ_____no_output_____Protocolは`worker`と`plan`の組みによって作成されます。`worker`は実際のワーカーでもワーカーIDでも構いません。`plan`はオブジェクトでもポインタでも構いません。_____no_output_____

import torch as th
import syft as sy
hook = sy.TorchHook(th)

# IMPORTANT: ローカルワーカーはクライアントワーカーにはなれません
hook.local_worker.is_client_worker = False_____no_output_____

Planを3つ定義して、Protocolとして纏めてみましょう。機能は全て1を足すオペレーションです。_____no_output_____

@sy.func2plan(args_shape=[(1,)])
def inc1(x):
    return x + 1

@sy.func2plan(args_shape=[(1,)])
def inc2(x):
    return x + 1

@sy.func2plan(args_shape=[(1,)])
def inc3(x):
    return x + 1

protocol = sy.Protocol([("worker1", inc1), ("worker2", inc2), ("worker3", inc3)])_____no_output_____

次にProtocolをワーカーにくっつける必要があります。これは`.deploy(*workers)`コマンドで実行できます。では、ワーカーを作成しましょう。_____no_output_____

bob = sy.VirtualWorker(hook, id="bob")
alice = sy.VirtualWorker(hook, id="alice")
charlie = sy.VirtualWorker(hook, id="charlie")_____no_output_____workers = alice, bob, charlie

protocol.deploy(*workers)_____no_output_____

各Planはそれぞれ関連づけられたワーカーの元に既に送られています。既にデプロイされているんです。

実は2つのフェーズから成り立っています。第一フェーズはワーカーとPlanの関連付けです。この段階ではワーカーの名前の文字列と関連づけられているだけです。第二フェーズでは、実際に各Planが各ワーカーの元へ送信されます。_____no_output_____### 2. Protocolの実行

Protocolを実行するというのは、全てのPlanを順番に実行する事と同義です。そのためにやるべき事は、最初のPlanが配置されている場所に入力データを送り込む事だけです。すると最初のPlanは実行され、その結果が次のPlanの入力データとなります。全てのPlanが順次実行され、最後のPlanが完了するとポインタが返されます。_____no_output_____

x = th.tensor([1.0])
ptr = protocol.run(x)
ptr_____no_output_____ptr.get()_____no_output_____

入力された1.0は3つのPlanを通過して3度1がたされ、4になって戻ってきています。_____no_output_____実は、引数にポインタを使ってPlanをリモート実行することも可能です。_____no_output_____

james = sy.VirtualWorker(hook, id="james")_____no_output_____protocol.send(james)_____no_output_____x = th.tensor([1.0]).send(james)
ptr = protocol.run(x)
ptr_____no_output_____

見ての通り、結果として返されるポインタはJamesがもっていますね。_____no_output_____

ptr = ptr.get()
ptr_____no_output_____ptr = ptr.get()
ptr_____no_output_____

### 3. Protocolの検索

実運用では、外部からProtocolをダウンロードしてきて、あなたのリモートワーカーにデプロイし、あなたのデータに対して実行したいと思うかもしれません。_____no_output_____それでは**まだデプロイされていない**Protocolを初期化して、リモートワーカーへ送ってみましょう。_____no_output_____

protocol = sy.Protocol([("worker1", inc1), ("worker2", inc2), ("worker3", inc3)])
protocol.tag('my_protocol')
protocol.send(james)_____no_output_____me = sy.hook.local_worker # ローカルワーカーをmeとして取得しておきます_____no_output_____

ここでProtocolを検索してみましょう。_____no_output_____

responses = me.request_search(['my_protocol'], location=james)
responses_____no_output_____

今、Protocolのポインタに対してアクセスできます。_____no_output_____

ptr_protocol = responses[0]_____no_output_____

他のポインタと同じように、実態を受け取る事も出来ます。_____no_output_____

protocol_back = ptr_protocol.get()
protocol_back_____no_output_____

後は先ほど学んだ手順と同様です。_____no_output_____

protocol_back.deploy(alice, bob, charlie)

x = th.tensor([1.0])
ptr = protocol_back.run(x)
ptr.get()_____no_output_____

Protocol関連のより実践的な例はこれから追加される予定ですが、Protocolが持つ可能性については十分に感じて頂けたと思います。_____no_output_____### PySyftのGitHubレポジトリにスターをつける

一番簡単に貢献できる方法はこのGitHubのレポジトリにスターを付けていただくことです。スターが増えると露出が増え、より多くのデベロッパーにこのクールな技術の事を知って貰えます。

- [Star PySyft](https://github.com/OpenMined/PySyft)

### Slackに入る

最新の開発状況のトラッキングする一番良い方法はSlackに入ることです。
下記フォームから入る事ができます。
[http://slack.openmined.org](http://slack.openmined.org)

### コードプロジェクトに参加する

コミュニティに貢献する一番良い方法はソースコードのコントリビューターになることです。PySyftのGitHubへアクセスしてIssueのページを開き、"Projects"で検索してみてください。参加し得るプロジェクトの状況を把握することができます。また、"good first issue"とマークされているIssueを探す事でミニプロジェクトを探すこともできます。

- [PySyft Projects](https://github.com/OpenMined/PySyft/issues?q=is%3Aopen+is%3Aissue+label%3AProject)
- [Good First Issue Tickets](https://github.com/OpenMined/PySyft/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)

### 寄付

もし、ソースコードで貢献できるほどの時間は取れないけど、是非何かサポートしたいという場合は、寄付をしていただくことも可能です。寄附金の全ては、ハッカソンやミートアップの開催といった、コミュニティ運営経費として利用されます。

[OpenMined's Open Collective Page](https://opencollective.com/openmined)_____no_output_____