# Notebook from shiling2007/Python-
Path: Understanding dynamic_factor default setting and mlemodel.ipynb

<code>
from IPython.core.display import display, HTML, Image
display(HTML("<style>.container { width:100% !important; }</style>"))
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
_____no_output_____%matplotlib inline

import numpy as np
import pandas as pd
import statsmodels.api as sm
import matplotlib.pyplot as plt

np.set_printoptions(precision=4, suppress=True, linewidth=120)_____no_output_____###simulate time series data
periods=200
seed=98165
# factor_model=pd.DataFrame(pd.date_range(start='1/1/2018', end='2/1/2019', freq='M', normalize=True),columns=['time']).astype('datetime64[M]')

factor_model=pd.DataFrame(pd.date_range(start='1/1/2006', periods=periods, freq='M', normalize=True),columns=['time']).astype('datetime64[M]')

np.random.seed(seed)

def factor_gen(size=periods, beta=0.8, burner=2, scale=1):
    j=burner+size
    x=np.random.normal(loc=0.0, scale=scale, size=j)

    y=[0]*j
    for i in range(j):

        if i==0:
            y0=0
        else:
            y[i]=beta*y0 + x[i]
            y0=y[i]

    return y[-size:], x[-size:]

###industry factor
fact_name='ind_fact'
fact_name_x='ind_x'
fact, fact_x = factor_gen(size=periods, beta=0.5, burner=200)
factor_model[fact_name]=fact
factor_model[fact_name_x]=fact_x

###segment factor
seg=2
sub=10
for i in range(seg):
    fact_name='seg_fact_'+str(i)
    fact_name_x='seg_x_'+str(i)
    fact, fact_x = factor_gen(size=periods, beta=0.7, burner=200)
    factor_model[fact_name]=fact
    factor_model[fact_name_x]=fact_x

###brand factor
for i in range(sub):
    fact_name='sub_fact_'+str(i)
    fact_name_x='sub_x_'+str(i)
    fact, fact_x = factor_gen(size=periods, beta=0.4, burner=200)
    factor_model[fact_name]=fact
    factor_model[fact_name_x]=fact_x

###idiosyncratic factor
for s in range(seg):
    for b in range(sub):
        fact_name='idi_fact_' + str(s) + '_' + str(b)
        fact_name_x='idi_fact_x' + str(s) + '_' + str(b)
        fact, fact_x = factor_gen(size=periods, beta=0.0, burner=200, scale=1)
        factor_model[fact_name]=fact
        factor_model[fact_name_x]=fact_x

factor_model['x1']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))
factor_model['x2']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))
factor_model['x3']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))
factor_model['x4']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))
factor_model['x5']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))
factor_model['x6']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))
factor_model['x7']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))
factor_model['x8']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))
factor_model['x9']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))
factor_model['x10']=pd.DataFrame(np.random.normal(loc=0.0, scale=1, size=periods))

###observed y --output

for s in range(seg):
    for b in range(sub):
        fact_name='y_' + str(s) + '_' + str(b)
        seg_fact='seg_fact_' + str(s)
        sub_fact='sub_fact_' + str(b)
        idi_fact= 'idi_fact_' + str(s) + '_' + str(b)
#         factor_model[fact_name]=factor_model['ind_fact']+factor_model[seg_fact] + factor_model[sub_fact] + factor_model[idi_fact]
#         factor_model[fact_name]=factor_model['ind_fact'] +factor_model[seg_fact] + factor_model[idi_fact] + 2*factor_model['x1'] -  2*factor_model['x2']
        factor_model[fact_name]=factor_model['ind_fact']  + factor_model[idi_fact] + 0.2*factor_model['x1'] -  0.2*factor_model['x2'] \
        + 0.2*factor_model['x3'] -  0.2*factor_model['x4'] + 0.2*factor_model['x5'] -  0.2*factor_model['x6'] \
        + 0.2*factor_model['x7'] -  0.2*factor_model['x8'] + 0.2*factor_model['x9'] -  0.2*factor_model['x10']
        factor_model[fact_name_x]=fact_x

# print(factor_model.columns)
_=factor_model[[name for name in factor_model.columns if 'y_' in name]][(
    factor_model.time.dt.year<=2010)].plot(subplots=True, layout=( seg,sub), figsize=(25, 5), grid=True, legend=False,
                                           title='All are similar in ind::Rows are similar in seg::Columns are similar in seg')
factor_model.head(5)_____no_output_____endog=factor_model[['y_0_0','y_0_1','y_0_2','y_1_0','y_1_2','y_1_3']]
exog=factor_model[['x1','x2']]_____no_output_____mod = sm.tsa.DynamicFactor(endog, exog=exog, k_factors=2, factor_order=1) #exog=exog1,_____no_output_____###take a look of this instance_____no_output_____list_terms=dir(mod)
pass_list=['__dict__','exog','endog']
for term in list_terms:
    if term not in pass_list:
        string='mod.'+term
        print('>>>> '+ string + '<<<<\n', eval(string))>>>> mod.__class__<<<<
 <class 'statsmodels.tsa.statespace.dynamic_factor.DynamicFactor'>
>>>> mod.__delattr__<<<<
 <method-wrapper '__delattr__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__dir__<<<<
 <built-in method __dir__ of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__doc__<<<<

    Dynamic factor model

    Parameters
    ----------
    endog : array_like
        The observed time-series process :math:`y`
    exog : array_like, optional
        Array of exogenous regressors for the observation equation, shaped
        nobs x k_exog.
    k_factors : int
        The number of unobserved factors.
    factor_order : int
        The order of the vector autoregression followed by the factors.
    error_cov_type : {'scalar', 'diagonal', 'unstructured'}, optional
        The structure of the covariance matrix of the observation error term,
        where "unstructured" puts no restrictions on the matrix, "diagonal"
        requires it to be any diagonal matrix (uncorrelated errors), and
        "scalar" requires it to be a scalar times the identity matrix. Default
        is "diagonal".
    error_order : int, optional
        The order of the vector autoregression followed by the observation
        error component. Default is None, corresponding to white noise errors.
    error_var : bool, optional
        Whether or not to model the errors jointly via a vector autoregression,
        rather than as individual autoregressions. Has no effect unless
        `error_order` is set. Default is False.
    enforce_stationarity : bool, optional
        Whether or not to transform the AR parameters to enforce stationarity
        in the autoregressive component of the model. Default is True.
    **kwargs
        Keyword arguments may be used to provide default values for state space
        matrices or for Kalman filtering options. See `Representation`, and
        `KalmanFilter` for more details.

    Attributes
    ----------
    exog : array_like, optional
        Array of exogenous regressors for the observation equation, shaped
        nobs x k_exog.
    k_factors : int
        The number of unobserved factors.
    factor_order : int
        The order of the vector autoregression followed by the factors.
    error_cov_type : {'diagonal', 'unstructured'}
        The structure of the covariance matrix of the error term, where
        "unstructured" puts no restrictions on the matrix and "diagonal"
        requires it to be a diagonal matrix (uncorrelated errors).
    error_order : int
        The order of the vector autoregression followed by the observation
        error component.
    error_var : bool
        Whether or not to model the errors jointly via a vector autoregression,
        rather than as individual autoregressions. Has no effect unless
        `error_order` is set.
    enforce_stationarity : bool, optional
        Whether or not to transform the AR parameters to enforce stationarity
        in the autoregressive component of the model. Default is True.

    Notes
    -----
    The dynamic factor model considered here is in the so-called static form,
    and is specified:

    .. math::

        y_t & = \Lambda f_t + B x_t + u_t \\
        f_t & = A_1 f_{t-1} + \dots + A_p f_{t-p} + \eta_t \\
        u_t & = C_1 u_{t-1} + \dots + C_1 f_{t-q} + \varepsilon_t

    where there are `k_endog` observed series and `k_factors` unobserved
    factors. Thus :math:`y_t` is a `k_endog` x 1 vector and :math:`f_t` is a
    `k_factors` x 1 vector.

    :math:`x_t` are optional exogenous vectors, shaped `k_exog` x 1.

    :math:`\eta_t` and :math:`\varepsilon_t` are white noise error terms. In
    order to identify the factors, :math:`Var(\eta_t) = I`. Denote
    :math:`Var(\varepsilon_t) \equiv \Sigma`.

    Options related to the unobserved factors:

    - `k_factors`: this is the dimension of the vector :math:`f_t`, above.
      To exclude factors completely, set `k_factors = 0`.
    - `factor_order`: this is the number of lags to include in the factor
      evolution equation, and corresponds to :math:`p`, above. To have static
      factors, set `factor_order = 0`.

    Options related to the observation error term :math:`u_t`:

    - `error_order`: the number of lags to include in the error evolution
      equation; corresponds to :math:`q`, above. To have white noise errors,
      set `error_order = 0` (this is the default).
    - `error_cov_type`: this controls the form of the covariance matrix
      :math:`\Sigma`. If it is "dscalar", then :math:`\Sigma = \sigma^2 I`. If
      it is "diagonal", then
      :math:`\Sigma = \text{diag}(\sigma_1^2, \dots, \sigma_n^2)`. If it is
      "unstructured", then :math:`\Sigma` is any valid variance / covariance
      matrix (i.e. symmetric and positive definite).
    - `error_var`: this controls whether or not the errors evolve jointly
      according to a VAR(q), or individually according to separate AR(q)
      processes. In terms of the formulation above, if `error_var = False`,
      then the matrices :math:C_i` are diagonal, otherwise they are general
      VAR matrices.

    References
    ----------
    .. [*] Lütkepohl, Helmut. 2007.
       New Introduction to Multiple Time Series Analysis.
       Berlin: Springer.

>>>> mod.__eq__<<<<
 <method-wrapper '__eq__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__format__<<<<
 <built-in method __format__ of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__ge__<<<<
 <method-wrapper '__ge__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__getattribute__<<<<
 <method-wrapper '__getattribute__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__getitem__<<<<
 <bound method MLEModel.__getitem__ of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.__gt__<<<<
 <method-wrapper '__gt__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__hash__<<<<
 <method-wrapper '__hash__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__init__<<<<
 <bound method DynamicFactor.__init__ of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.__init_subclass__<<<<
 <built-in method __init_subclass__ of type object at 0x000001DD76520798>
>>>> mod.__le__<<<<
 <method-wrapper '__le__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__lt__<<<<
 <method-wrapper '__lt__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__module__<<<<
 statsmodels.tsa.statespace.dynamic_factor
>>>> mod.__ne__<<<<
 <method-wrapper '__ne__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__new__<<<<
 <built-in method __new__ of type object at 0x0000000070A3BD90>
>>>> mod.__reduce__<<<<
 <built-in method __reduce__ of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__reduce_ex__<<<<
 <built-in method __reduce_ex__ of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__repr__<<<<
 <method-wrapper '__repr__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__setattr__<<<<
 <method-wrapper '__setattr__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__setitem__<<<<
 <bound method MLEModel.__setitem__ of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.__sizeof__<<<<
 <built-in method __sizeof__ of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__str__<<<<
 <method-wrapper '__str__' of DynamicFactor object at 0x000001DD7A768E10>
>>>> mod.__subclasshook__<<<<
 <built-in method __subclasshook__ of type object at 0x000001DD76520798>
>>>> mod.__weakref__<<<<
 None
>>>> mod._clone_from_init_kwds<<<<
 <bound method MLEModel._clone_from_init_kwds of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._data_attr<<<<
 ['exog', 'endog', 'data.exog', 'data.endog', 'data.orig_endog', 'data.orig_exog']
>>>> mod._error_order<<<<
 0
>>>> mod._factor_order<<<<
 2
>>>> mod._fit_collinear<<<<
 <bound method LikelihoodModel._fit_collinear of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._fit_zeros<<<<
 <bound method LikelihoodModel._fit_zeros of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._fixed_params<<<<
 None
>>>> mod._fixed_params_index<<<<
 None
>>>> mod._forecasts_error_partial_derivatives<<<<
 <bound method MLEModel._forecasts_error_partial_derivatives of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._formula_max_endog<<<<
 1
>>>> mod._free_params_index<<<<
 None
>>>> mod._get_exog_names<<<<
 <bound method TimeSeriesModel._get_exog_names of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._get_extension_time_varying_matrices<<<<
 <bound method MLEModel._get_extension_time_varying_matrices of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._get_index_label_loc<<<<
 <bound method TimeSeriesModel._get_index_label_loc of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._get_index_loc<<<<
 <bound method TimeSeriesModel._get_index_loc of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._get_index_with_final_state<<<<
 <bound method MLEModel._get_index_with_final_state of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._get_init_kwds<<<<
 <bound method MLEModel._get_init_kwds of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._get_prediction_index<<<<
 <bound method TimeSeriesModel._get_prediction_index of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._handle_data<<<<
 <bound method Model._handle_data of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._has_fixed_params<<<<
 False
>>>> mod._hessian_complex_step<<<<
 <bound method MLEModel._hessian_complex_step of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._hessian_finite_difference<<<<
 <bound method MLEModel._hessian_finite_difference of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._hessian_oim<<<<
 <bound method MLEModel._hessian_oim of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._hessian_opg<<<<
 <bound method MLEModel._hessian_opg of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._hessian_param_defaults<<<<
 [True, 'approx', None, False]
>>>> mod._hessian_param_names<<<<
 ['transformed', 'hessian_method', 'approx_complex_step', 'approx_centered']
>>>> mod._idx_error_cov<<<<
 ('obs_cov', array([0, 1, 2, 3, 4, 5]), array([0, 1, 2, 3, 4, 5]))
>>>> mod._idx_error_transition<<<<
 ('transition', slice(0, 0, None), slice(0, 0, None))
>>>> mod._idx_exog<<<<
 ('obs_intercept', slice(None, 6, None), slice(None, None, None))
>>>> mod._idx_factor_transition<<<<
 ('transition', slice(None, 2, None), slice(None, 2, None))
>>>> mod._idx_loadings<<<<
 ('design', slice(None, None, None), slice(None, 2, None))
>>>> mod._index<<<<
 RangeIndex(start=0, stop=200, step=1)
>>>> mod._index_dates<<<<
 False
>>>> mod._index_freq<<<<
 None
>>>> mod._index_generated<<<<
 False
>>>> mod._index_inferred_freq<<<<
 False
>>>> mod._index_int64<<<<
 False
>>>> mod._index_none<<<<
 False
>>>> mod._init_dates<<<<
 <bound method TimeSeriesModel._init_dates of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._init_keys<<<<
 ['k_factors', 'factor_order', 'error_order', 'error_var', 'error_cov_type', 'enforce_stationarity', 'initialization']
>>>> mod._init_kwargs<<<<
 {'k_posdef': 2, 'initialization': 'stationary'}
>>>> mod._initialize_error_cov<<<<
 <bound method DynamicFactor._initialize_error_cov of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._initialize_error_cov_diagonal<<<<
 <bound method DynamicFactor._initialize_error_cov_diagonal of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._initialize_error_cov_unstructured<<<<
 <bound method DynamicFactor._initialize_error_cov_unstructured of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._initialize_error_transition<<<<
 <bound method DynamicFactor._initialize_error_transition of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._initialize_error_transition_individual<<<<
 <bound method DynamicFactor._initialize_error_transition_individual of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._initialize_error_transition_var<<<<
 <bound method DynamicFactor._initialize_error_transition_var of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._initialize_error_transition_white_noise<<<<
 <bound method DynamicFactor._initialize_error_transition_white_noise of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._initialize_exog<<<<
 <bound method DynamicFactor._initialize_exog of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._initialize_factor_transition<<<<
 <bound method DynamicFactor._initialize_factor_transition of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._initialize_loadings<<<<
 <bound method DynamicFactor._initialize_loadings of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._loglike_param_defaults<<<<
 [True, False, False]
>>>> mod._loglike_param_names<<<<
 ['transformed', 'includes_fixed', 'complex_step']
>>>> mod._params_error_cov<<<<
 slice(24, 30, None)
>>>> mod._params_error_transition<<<<
 slice(34, 34, None)
>>>> mod._params_exog<<<<
 slice(12, 24, None)
>>>> mod._params_factor_transition<<<<
 slice(30, 34, None)
>>>> mod._params_index<<<<
 None
>>>> mod._params_loadings<<<<
 slice(0, 12, None)
>>>> mod._res_classes<<<<
 {'fit': (<class 'statsmodels.tsa.statespace.dynamic_factor.DynamicFactorResults'>, <class 'statsmodels.tsa.statespace.dynamic_factor.DynamicFactorResultsWrapper'>)}
>>>> mod._score_complex_step<<<<
 <bound method MLEModel._score_complex_step of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._score_finite_difference<<<<
 <bound method MLEModel._score_finite_difference of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._score_harvey<<<<
 <bound method MLEModel._score_harvey of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._score_obs_harvey<<<<
 <bound method MLEModel._score_obs_harvey of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._score_param_defaults<<<<
 [True, False, 'approx', None, False]
>>>> mod._score_param_names<<<<
 ['transformed', 'includes_fixed', 'score_method', 'approx_complex_step', 'approx_centered']
>>>> mod._set_exog_names<<<<
 <bound method TimeSeriesModel._set_exog_names of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._unused_state<<<<
 False
>>>> mod._validate_can_fix_params<<<<
 <bound method DynamicFactor._validate_can_fix_params of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._validate_out_of_sample_exog<<<<
 <bound method MLEModel._validate_out_of_sample_exog of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod._wrap_results<<<<
 <bound method MLEModel._wrap_results of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.clone<<<<
 <bound method DynamicFactor.clone of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.data<<<<
 <statsmodels.base.data.PandasData object at 0x000001DD7A768C18>
>>>> mod.endog_names<<<<
 ['y_0_0', 'y_0_1', 'y_0_2', 'y_1_0', 'y_1_2', 'y_1_3']
>>>> mod.enforce_stationarity<<<<
 True
>>>> mod.error_cov_type<<<<
 diagonal
>>>> mod.error_order<<<<
 0
>>>> mod.error_var<<<<
 False
>>>> mod.exog_names<<<<
 ['x1', 'x2']
>>>> mod.factor_order<<<<
 1
>>>> mod.filter<<<<
 <bound method MLEModel.filter of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.fit<<<<
 <bound method MLEModel.fit of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.fit_constrained<<<<
 <bound method MLEModel.fit_constrained of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.fix_params<<<<
 <bound method MLEModel.fix_params of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.from_formula<<<<
 <bound method MLEModel.from_formula of <class 'statsmodels.tsa.statespace.dynamic_factor.DynamicFactor'>>
>>>> mod.handle_params<<<<
 <bound method MLEModel.handle_params of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.hessian<<<<
 <bound method MLEModel.hessian of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.impulse_responses<<<<
 <bound method MLEModel.impulse_responses of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.information<<<<
 <bound method LikelihoodModel.information of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.initial_variance<<<<
 1000000.0
>>>> mod.initialization<<<<
 <statsmodels.tsa.statespace.initialization.Initialization object at 0x000001DD6CB4DEF0>
>>>> mod.initialize<<<<
 <bound method LikelihoodModel.initialize of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.initialize_approximate_diffuse<<<<
 <bound method MLEModel.initialize_approximate_diffuse of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.initialize_known<<<<
 <bound method MLEModel.initialize_known of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.initialize_statespace<<<<
 <bound method MLEModel.initialize_statespace of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.initialize_stationary<<<<
 <bound method MLEModel.initialize_stationary of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.k_constant<<<<
 0
>>>> mod.k_endog<<<<
 6
>>>> mod.k_exog<<<<
 2
>>>> mod.k_factors<<<<
 2
>>>> mod.k_params<<<<
 34
>>>> mod.k_states<<<<
 2
>>>> mod.loglike<<<<
 <bound method MLEModel.loglike of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.loglikelihood_burn<<<<
 0
>>>> mod.loglikeobs<<<<
 <bound method MLEModel.loglikeobs of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.mle_regression<<<<
 True
>>>> mod.nobs<<<<
 200
>>>> mod.observed_information_matrix<<<<
 <bound method MLEModel.observed_information_matrix of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.opg_information_matrix<<<<
 <bound method MLEModel.opg_information_matrix of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.param_names<<<<
 ['loading.f1.y_0_0', 'loading.f2.y_0_0', 'loading.f1.y_0_1', 'loading.f2.y_0_1', 'loading.f1.y_0_2', 'loading.f2.y_0_2', 'loading.f1.y_1_0', 'loading.f2.y_1_0', 'loading.f1.y_1_2', 'loading.f2.y_1_2', 'loading.f1.y_1_3', 'loading.f2.y_1_3', 'beta.x1.y_0_0', 'beta.x2.y_0_0', 'beta.x1.y_0_1', 'beta.x2.y_0_1', 'beta.x1.y_0_2', 'beta.x2.y_0_2', 'beta.x1.y_1_0', 'beta.x2.y_1_0', 'beta.x1.y_1_2', 'beta.x2.y_1_2', 'beta.x1.y_1_3', 'beta.x2.y_1_3', 'sigma2.y_0_0', 'sigma2.y_0_1', 'sigma2.y_0_2', 'sigma2.y_1_0', 'sigma2.y_1_2', 'sigma2.y_1_3', 'L1.f1.f1', 'L1.f2.f1', 'L1.f1.f2', 'L1.f2.f2']
>>>> mod.parameters<<<<
 {'factor_loadings': 12, 'exog': 12, 'error_cov': 6, 'factor_transition': 4, 'error_transition': 0}
>>>> mod.predict<<<<
 <bound method Model.predict of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.prepare_data<<<<
 <bound method MLEModel.prepare_data of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.score<<<<
 <bound method MLEModel.score of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.score_obs<<<<
 <bound method MLEModel.score_obs of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.set_conserve_memory<<<<
 <bound method MLEModel.set_conserve_memory of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.set_filter_method<<<<
 <bound method MLEModel.set_filter_method of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.set_inversion_method<<<<
 <bound method MLEModel.set_inversion_method of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.set_smoother_output<<<<
 <bound method MLEModel.set_smoother_output of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.set_stability_method<<<<
 <bound method MLEModel.set_stability_method of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.simulate<<<<
 <bound method MLEModel.simulate of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.simulation_smoother<<<<
 <bound method MLEModel.simulation_smoother of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.smooth<<<<
 <bound method MLEModel.smooth of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.ssm<<<<
 <statsmodels.tsa.statespace.simulation_smoother.SimulationSmoother object at 0x000001DD7A768438>
>>>> mod.start_params<<<<
 [-17.5078   0.1382 -17.1628 -12.9134 -18.4987  -0.715  -18.3622  -0.9776 -18.4136   4.4369 -18.1275   8.9897   0.1509
  -0.1713   0.4105  -0.1718   0.325   -0.2131   0.2766  -0.0934   0.3476  -0.1121   0.1206  -0.0266   2.1256   2.1807
   2.1943   2.3621   2.2545   2.3249   0.2905  -0.0569  -0.0363  -0.0118]
>>>> mod.state_names<<<<
 ['f1', 'f2']
>>>> mod.tolerance<<<<
 1e-19
>>>> mod.transform_jacobian<<<<
 <bound method MLEModel.transform_jacobian of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.transform_params<<<<
 <bound method DynamicFactor.transform_params of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.untransform_params<<<<
 <bound method DynamicFactor.untransform_params of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
>>>> mod.update<<<<
 <bound method DynamicFactor.update of <statsmodels.tsa.statespace.dynamic_factor.DynamicFactor object at 0x000001DD7A768E10>>
list_matrix=['state_cov','obs_cov','state_intercept','obs_intercept','design','transition','selection','err_transition']

for name in list_matrix:
    try:
        print(name,'\n', mod.ssm[name],'\n',mod.ssm[name].shape)
    except:
        pass
state_cov
 [[1. 0.]
 [0. 1.]]
 (2, 2)
obs_cov
 [[0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0.]]
 (6, 6)
state_intercept
 [0. 0.]
 (2,)
obs_intercept
 [[0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]]
 (6, 200)
design
 [[0. 0.]
 [0. 0.]
 [0. 0.]
 [0. 0.]
 [0. 0.]
 [0. 0.]]
 (6, 2)
transition
 [[0. 0.]
 [0. 0.]]
 (2, 2)
selection
 [[1. 0.]
 [0. 1.]]
 (2, 2)
initial_res = mod.fit(method='powell', disp=True)
res = mod.fit(initial_res.params, disp=True, maxiter=5000)Optimization terminated successfully.
         Current function value: 9.451653
         Iterations: 14
         Function evaluations: 5035
np.set_printoptions(precision=4, suppress=True, linewidth=120)
list_matrix=['state_cov','obs_cov','state_intercept','obs_intercept','design','transition','selection','err_transition']

for name in list_matrix:
    try:
        print(name,'\n', mod.ssm[name],'\n',mod.ssm[name].shape)
    except:
        pass
res.paramsstate_cov
 [[1. 0.]
 [0. 1.]]
 (2, 2)
obs_cov
 [[0.     0.     0.     0.     0.     0.    ]
 [0.     1.2573 0.     0.     0.     0.    ]
 [0.     0.     0.7701 0.     0.     0.    ]
 [0.     0.     0.     1.0255 0.     0.    ]
 [0.     0.     0.     0.     0.8688 0.    ]
 [0.     0.     0.     0.     0.     0.9161]]
 (6, 6)
state_intercept
 [0. 0.]
 (2,)
obs_intercept
 [[-0.1119 -0.0744 -0.0187 ...  0.229  -0.1854 -0.3235]
 [-0.3427 -0.4121 -0.0327 ...  0.4412 -0.201  -0.8479]
 [-0.2711 -0.2806 -0.0319 ...  0.4133 -0.2495 -0.7061]
 [-0.1998 -0.2605 -0.0164 ...  0.2287 -0.0769 -0.4787]
 [-0.2578 -0.3363 -0.0211 ...  0.295  -0.0991 -0.6176]
 [-0.0475 -0.0833 -0.001  ...  0.0241  0.0244 -0.0971]]
 (6, 200)
design
 [[-1.427  -0.2189]
 [-0.8976  0.4218]
 [-0.9439  0.6993]
 [-0.9941  0.5546]
 [-0.9628  0.6584]
 [-1.0338  0.5801]]
 (6, 2)
transition
 [[ 0.2666 -0.0763]
 [-0.0967  0.0391]]
 (2, 2)
selection
 [[1. 0.]
 [0. 1.]]
 (2, 2)
df_coef=pd.DataFrame(res.params)_____no_output_____x_intercept=pd.DataFrame(mod.ssm['obs_intercept'].T,columns=['x1','x2','x3','x4','x5','x6'])
exog['x1_e']=df_coef[df_coef.index=='beta.x1.y_0_0'][0].values*exog['x1'] + df_coef[df_coef.index=='beta.x2.y_0_0'][0].values*exog['x2']
exog['x1_m']=x_intercept['x1']C:\Users\jshi2\AppData\Local\Continuum\Anaconda3\lib\site-packages\ipykernel_launcher.py:2: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy

C:\Users\jshi2\AppData\Local\Continuum\Anaconda3\lib\site-packages\ipykernel_launcher.py:3: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  This is separate from the ipykernel package so we can avoid doing imports until
exog_____no_output_____res.summary(separate_params=False)_____no_output_____
</code>

## Extended the general model with fixed parameters _____no_output_____

<code>
endog=factor_model[['y_0_0','y_0_1','y_0_2','y_1_0','y_1_2','y_1_3']]
exog=factor_model[['x1','x2']]_____no_output_____# case 0) reproduce the results with no constrainsts
from statsmodels.tsa.statespace import tools
class ExtendedDFM(sm.tsa.DynamicFactor):
    def __init__(self, endog, exog=exog,  **kwargs):
            # Setup the model as if we had a factor order of 4
            super(ExtendedDFM, self).__init__(
                endog,  exog=exog, k_factors=2, factor_order=1, **kwargs)
_____no_output_____mod_ext = ExtendedDFM(endog, exog=exog)
initial_res_ext = mod_ext.fit(method='powell', disp=True)
res_ext = mod_ext.fit(initial_res_ext.params, disp=True, maxiter=5000)
res_ext.summary(separate_params=False)Optimization terminated successfully.
         Current function value: 9.451653
         Iterations: 14
         Function evaluations: 5035
#case 1) fixed all error cov as 1

from statsmodels.tsa.statespace import tools
class ExtendedDFM(sm.tsa.DynamicFactor):
    def __init__(self, endog, exog=exog,  **kwargs):
            # Setup the model as if we had a factor order of 4
            super(ExtendedDFM, self).__init__(
                endog,  exog=exog, k_factors=2, factor_order=1, **kwargs)

    def update(self, params, transformed=True, **kwargs):
        params[self._params_error_cov] = 1
        super(ExtendedDFM, self).update(params, transformed=True, **kwargs)

mod_ext = ExtendedDFM(endog, exog=exog)
initial_res_ext = mod_ext.fit(method='powell', disp=True)
res_ext = mod_ext.fit(initial_res_ext.params, disp=True, maxiter=5000)
res_ext.summary(separate_params=False)Optimization terminated successfully.
         Current function value: 9.464151
         Iterations: 11
         Function evaluations: 3889
#case 2) fixed all error cov as 1 + off diagonal in factor transition as 0

from statsmodels.tsa.statespace import tools
class ExtendedDFM(sm.tsa.DynamicFactor):
    def __init__(self, endog, exog=exog,  **kwargs):
            # Setup the model as if we had a factor order of 4
            super(ExtendedDFM, self).__init__(
                endog,  exog=exog, k_factors=2, factor_order=1, **kwargs)

    def update(self, params, transformed=True, **kwargs):
        params[self._params_error_cov] = 1
        params[31:33] = 0

        super(ExtendedDFM, self).update(params, transformed=True, **kwargs)

mod_ext = ExtendedDFM(endog, exog=exog)
# initial_res_ext = mod_ext.fit(method='powell', disp=True)
init=[0.2]*31
init2=[0.0001,0.001,0.0001]
init=init.append(init2)
res_ext = mod_ext.fit(init, disp=True, maxiter=5000)
res_ext.summary(separate_params=False)C:\Users\jshi2\AppData\Local\Continuum\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\mlemodel.py:2937: RuntimeWarning: divide by zero encountered in true_divide
  return self.params / self.bse
C:\Users\jshi2\AppData\Local\Continuum\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\mlemodel.py:2937: RuntimeWarning: invalid value encountered in true_divide
  return self.params / self.bse
#case 3)  off diagonal in factor transition as 0

from statsmodels.tsa.statespace import tools
class ExtendedDFM(sm.tsa.DynamicFactor):
    def __init__(self, endog, exog=exog,  **kwargs):
            # Setup the model as if we had a factor order of 4
            super(ExtendedDFM, self).__init__(
                endog,  exog=exog, k_factors=2, factor_order=1, **kwargs)

    def update(self, params, transformed=True, **kwargs):
#         params[self._params_error_cov] = 1
        params[31:33] = 0

        super(ExtendedDFM, self).update(params, transformed=True, **kwargs)

mod_ext = ExtendedDFM(endog, exog=exog)
# initial_res_ext = mod_ext.fit(method='powell', disp=True)
init=[0.2]*31
init2=[0.0001,0.001,0.0001]
init=init.append(init2)
res_ext = mod_ext.fit(init, disp=True, maxiter=5000)
res_ext.summary(separate_params=False)C:\Users\jshi2\AppData\Local\Continuum\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\mlemodel.py:2937: RuntimeWarning: invalid value encountered in true_divide
  return self.params / self.bse
#case 4)  off diagonal in factor transition as 0 + set beta 0

from statsmodels.tsa.statespace import tools
class ExtendedDFM(sm.tsa.DynamicFactor):
    def __init__(self, endog, exog=exog,  **kwargs):
            # Setup the model as if we had a factor order of 4
            super(ExtendedDFM, self).__init__(
                endog,  exog=exog, k_factors=2, factor_order=1, **kwargs)

    def update(self, params, transformed=True, **kwargs):
#         params[self._params_error_cov] = 1
        params[31:33] = 0
        params[13] = 0

        super(ExtendedDFM, self).update(params, transformed=True, **kwargs)

mod_ext = ExtendedDFM(endog, exog=exog)
# initial_res_ext = mod_ext.fit(method='powell', disp=True)
init=[0.2]*31
init2=[0.0001,0.001,0.0001]
init=init.append(init2)
res_ext = mod_ext.fit(init, disp=True, maxiter=5000)
res_ext.summary(separate_params=False)C:\Users\jshi2\AppData\Local\Continuum\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\mlemodel.py:2937: RuntimeWarning: invalid value encountered in true_divide
  return self.params / self.bse

</code>


# Notebook from ilam56/wg-evolution
Path: implementations/notebooks_df/code_changes_lines.ipynb

## Code_Changes_Lines
This is the reference implementation for [Code_Changes_Lines](https://github.com/chaoss/wg-evolution/blob/master/metrics/Code_Changes_Lines.md),
a metric specified by the
[Evolution Working Group](https://github.com/chaoss/wg-evolution) of the
[CHAOSS project](https://chaoss.community).

Have a look at [README.md](../README.md) to find out how to run this notebook (and others in this directory) as well as to get a better understanding of the purpose of the implementations.

The implementation is described in two parts (see below):

* Class for computing Code_Changes_Lines
* An explanatory analysis of the class' functionality

Some more auxiliary information in this notebook:

* Examples of the use of the implementation_____no_output_____As discussed in the [README](../README.md) file, the scripts required to analyze the data fetched by Perceval are located in the `code_df` package. Due to python's import system, to import modules from a package which is not in the current directory, we have to either add the package to `PYTHONPATH` or simply append a `..` to `sys.path`, so that `code_df` can be successfully imported. _____no_output_____

from datetime import datetime
import matplotlib.pyplot as plt
import sys
sys.path.append('..')
from code_df import utils
from code_df import conditions
from code_df.commit import Commit
%matplotlib inline_____no_output_____class CodeChangesLines(Commit):
    """
    Class for Code_Changes_Lines
    """

    def _flatten(self, item):
        """
        Flatten a raw commit fetched by Perceval into a flat dictionary.

        A list with a single flat directory will be returned.
        That dictionary will have the elements we need for computing metrics.
        The list may be empty, if for some reason the commit should not
        be considered.

        :param item: raw item fetched by Perceval (dictionary)
        :returns:    list of a single flat dictionary
        """

        creation_date = utils.str_to_date(item['data']['AuthorDate'])
        if self.since and (self.since > creation_date):
            return []

        if self.until and (self.until < creation_date):
            return []

        code_files = [file['file'] for file in item['data']['files'] if
                      all(condition.check(file['file'])
                          for condition in self.is_code)]

        if len(code_files) > 0:
            flat = {
                'repo': item['origin'],
                'hash': item['data']['commit'],
                'author': item['data']['Author'],
                'category': "commit",
                'created_date': creation_date,
                'committer': item['data']['Commit'],
                'commit_date': utils.str_to_date(item['data']['CommitDate']),
                'files_no': len(item['data']['files']),
                'refs': item['data']['refs'],
                'parents': item['data']['parents'],
                'files': item['data']['files']
            }

            # actions
            actions = 0
            for file in item['data']['files']:
                if 'action' in file:
                    actions += 1
            flat['files_action'] = actions

            # Merge commit check
            if 'Merge' in item['data']:
                flat['merge'] = True
            else:
                flat['merge'] = False

            # modifications
            modified_lines = 0
            for file in item['data']['files']:
                if 'added' and 'removed' in file:
                    try:
                        modified_lines += int(file['added']) + int(file['removed'])

                    except ValueError:
                        # in case of compressed files,
                        # additions and deletions are "-"
                        pass

            flat['modifications'] = modified_lines

            return [flat]
        else:
            return []

    def compute(self):
        """
        Compute the number of lines modified in the data fetched
        by Perceval.

        It computes the sum of the 'modifications' column
        in the DataFrame.

        :returns modifications_count: The total number of
            lines modified (int)
        """

        df = self.df
        modifications_count = df['modifications'].sum()

        return modifications_count

    def _agg(self, df, period):
        """
        Perform an aggregation operation on a DataFrame or Series
        to find the total number of lines modified in a every interval
        of the period specified in the time_series method, like
        'M', 'W',etc.

        It adds the number of lines modified for every row in the
        series.

        :param df: a pandas DataFrame on which the aggregation will be
            applied.

        :param period: A string which can be any one of the pandas time
            series rules:
            'W': week
            'M': month
            'D': day

        :returns df: The aggregated dataframe, where aggregations have
            been performed on the "modifications"
        """

        df = df.resample(period)['modifications'].agg(['sum'])

        return df_____no_output_____

## Performing the Analysis
Using the above class, we can perform several kinds of analysis on the JSON data file, fetched by Perceval.
For starters, we can perform a simple calculation of the number of modified lines (additions plus deletions) in the file. To make things simple, we will use the `Naive` implementation for deciding whether a given commit affects the source code or not. Again, the naive implementation assumes that all files are part of the source code, and hence, all commits are considered to affect it. The `Naive` implementation is the default option. _____no_output_____### Counting the total number of modified lines
We first read the JSON file containing Perceval data using the `read_json_file` utility function. _____no_output_____

items = utils.read_json_file('../git-commits.json')_____no_output_____

Let's use the `compute` method to count the total number of lines modified after instantiating the above class.

Notice that here, we are redefining the `_flatten` method of the `Commit` class, the parent of the `CodeChangesLines` class. The reason for doing this is to add a `modifications` column to the dataframe. This makes it easier to compute this metric.

First, we will do the computation without passing any since and until dates. Next, we can pass in the start and end dates as a tuple. The format would be `%Y-%m-%d`._____no_output_____

changes = CodeChangesLines(items)
print("The total number of modified lines "
      "in the file is {}.".format(changes.compute()))

date_since = datetime.strptime("2018-01-01", "%Y-%m-%d")
date_until = datetime.strptime("2018-07-01", "%Y-%m-%d")

changes_dated = CodeChangesLines(items,
                        date_range=(date_since, date_until))

print("The total number of lines modified between "
      "2018-01-01 and 2018-07-01 is {}.".format(changes_dated.compute()))The total number of modified lines in the file is 563670.
The total number of lines modified between 2018-01-01 and 2018-07-01 is 192972.

### Counting the total number of lines modified by commits excluding merge commits
Moving on, lets make use of the `EmptyExclude` and `MergeExclude` classes to filter out empty and merge commits respectively. These classes are sub-classes of the `Commit` class in the `conditions` module. They provide two methods: `check()` and `set_commits`.

The `set_commits` method selects commits which satisfy a given condition (like excluding empty commits, for example) and stores the hashes of those commits in the set `included`, an instance variable of all `Commit` classes.

The `check()` method checks each commit in the DataFrame created from Perceval data and drops those rows which correspond to commits not in `included`. _____no_output_____

changes_non_merge = CodeChangesLines(items,
                                  (date_since, date_until),
                                  conds=[conditions.MergeExclude()])
print("The total number of lines modified by non-merge commits between"
      " 2018-01-01 and 2018-07-01 is {}.".format(changes_non_merge.compute()))The total number of lines modified by non-merge commits between 2018-01-01 and 2018-07-01 is 94047.

## Counting the number of lines modified over regular time intervals
Using the `time_series` method, it is possible to compute the number of lines modified every month, or every week. This kind of analysis is useful in finding trends over time, as we will see in the cell below.

Let's perform a basic analysis: lets see the change in the number of lines modified between the same dates we used above on a weekly basis: 2018-01-01 and 2018-07-01.
The Code_Changes_Lines object, `changes_dated`, will be the same as used above._____no_output_____

weekly_df = changes_dated.time_series(period='W')_____no_output_____

Lets see what the dataframe returned by `time_series` looks like. As you will notice, the dataframe has rows corresponding to each and every week between the start and end dates. To do this, we simply set the `created_date` column of the DataFrame `changes_dated.df`, as its index and then `resample` it to whatever time period we need. In this case, we have used `W`. We will apply the 'sum' aggregation function on both the 'additions' and 'deletions' columns of the dataframe._____no_output_____

weekly_df_____no_output_____

Lets plot the dataframe `weekly_df` using matplotlib.pyplot. We use the `seaborn` theme and plot a simple line plot --- lines modified vs time interval. Using the `plt.fill_between` method allows us to "fill up" the area between the line plots and the x axis._____no_output_____

plt.figure(figsize=[15, 5])
plt.style.use('seaborn')
plt.plot(weekly_df['sum'])

plt.fill_between(y1=weekly_df['sum'], y2=0, x=weekly_df.index)
plt.title("Lines modified");
plt.show()_____no_output_____

The same thing can be tried for months, instead of weeks. By passing `month` in place of week, we get a similar dataframe but with only a few rows, due to the larger timescale. _____no_output_____## Counting line modifications by commits only made on the master branch
Another option one has while using this class for analyzing git commit data is to include only those commits for analysis which are on the master branch. To do this, we pass in an object of the `MasterInclude` class as a list to the `conds` parameter while instantiating the `CodeChangesGit` class.

We compute the number of commits created on the master branch after `2018-01-01`, which we stored in the `datetime` object, `date_since`. _____no_output_____

changes_only_master = CodeChangesLines(items,
                                    date_range=(date_since, None),
                                    conds=[conditions.MasterInclude()])

print("The total number of lines modified (additions, deletions) by commits made on the master branch "
      "after 2018-01-01 is {}.".format(changes_only_master.compute()))The total number of lines modified (additions, deletions) by commits made on the master branch after 2018-01-01 is 215191.

Lets do one last thing: the same thing we did in the cell above, but without including empty commits. In this case, we would also need to pass a `conditions.ExcludeEmpty` object to `conds`.

Also, lets exclude those commits which work solely on `markdown` files. We use the `PostfixExclude` class, a sub-class of `Code` for this._____no_output_____

changes_non_empty_master = CodeChangesLines(items,
                                          is_code=[conditions.PostfixExclude(postfixes=['.md'])],
                                          conds=[conditions.MasterInclude(), conditions.EmptyExclude()])

print("The total number of lines modified by non-empty commits made on the master branch is: {}".format(changes_non_empty_master.compute()))The total number of lines modified by non-empty commits made on the master branch is: 9226