DATA ANALYSIS USING PANDAS¶
In [ ]:
# DATA ANALYSIS USING PANDAS
# Analysis of the protein database (Result_protein.txt)
# that has been previously preprocessed using Python
# (Orginal data from the protein data bank: https://www.rcsb.org/)
In [ ]:
# Pandas is built on Numpy.
# The main difference between Numpy and Pandas are the indexes and labels
#Documentation: https://pandas.pydata.org/pandas-docs/stable/getting_started/10min.html
In [2]:
import pandas as pd
import numpy as np
import sys
In [ ]:
# READ
# There are several readers and writers available (CSV; JSON; html ; SAS; SQL; ...)
# Readers: read_csv ; read_sas ; ...etc.
# Writers: to_csv; to_sql ; to_sas ; ...etc
# Specify the absence of header: pd.read_csv(file, header=None)
# Generate a header: pd.read_csv(file, names=[n1,n2,n3])
In [3]:
df =pd.read_csv('Result_protein.txt', sep=" " )
Explore de data: head, info, columns, count, shape, describe, unique¶
In [4]:
df.head(3)
Out[4]:
In [9]:
df.info()
In [10]:
df.columns
Out[10]:
In [11]:
df.count()
Out[11]:
In [22]:
df.shape
Out[22]:
In [28]:
df.describe()
Out[28]:
In [19]:
df['family'].unique() #Get unique values of a column
Out[19]:
In [20]:
len(df['family'].unique())
Out[20]:
Filter specific values: str.contains, loc, iloc, isin¶
In [24]:
#Select a column
#df['type'] # Select 'type' column
#df[0:3] # Select rows from 0 to 3
df['type'][0:3] #Select 'type' column, rows from 0 to 3
Out[24]:
In [46]:
df[df['type'].str.contains('pro')][0:3]
Out[46]:
In [4]:
len(df[df['type'].str.contains('pro')])
Out[4]:
In [ ]:
#Selection by label
df.loc[]
#Selection by position
df.iloc[]
In [14]:
#df.loc[df['column_name'] == some_value]
df.loc[df['pdbid'] == '1dy3']
Out[14]:
In [29]:
df.iloc[[1,2,3,4],:]
Out[29]:
In [49]:
# Boolean Indexing
#df[df.%npolar > 0.4] Invalid syntax:
df[df.iloc[:,8]>0.40][0:3] #[0:3] limits the amount of output to display
Out[49]:
In [48]:
#[df.npolar>40 #returns a true/false list
df[df.npolar>40][0:3]
Out[48]:
In [60]:
df[df['chain'].isin(['A', 'B'])][0:6]
#df.isin({'chain': "A", 'type': "protein"})
#df2[df2['E'].isin(['two', 'four'])]
#countries = ["T4_LYSOZIME","MYOGLOBIN"]
#df.family.isin(countries)
Out[60]:
In [91]:
familyname = ["T4_LYSOZYME","MYOGLOBIN"]
len(df[df.family.isin(familyname)])
df[df.family.isin(familyname)][0:5]
Out[91]:
In [92]:
#not in
familyname = ["T4_LYSOZYME","MYOGLOBIN"]
df[~df.family.isin(familyname)][0:6]
Out[92]:
Dealing with missing data: dropna, fillna¶
In [103]:
# [In] len(df[df['family'].str.contains('MYOGLOBIN')])
# [Out] cannot index with vector containing NA / NaN values
# How to deal with missing data?
In [ ]:
# 1.- How to identify missing values??
In [95]:
df.info() #family contains null elements
df.isnull().any() # family true: means it contains null elements
Out[95]:
In [98]:
df.isnull().sum() #isnull returns a true/false matrix
Out[98]:
In [114]:
null_data = df[df.isnull().any(axis=1)] #print null rows
null_data.head()
Out[114]:
In [ ]:
# 2.- dropna --- returns a serie of non-null data : drop
In [115]:
len(df['type'].str.contains('protein'))
Out[115]:
In [117]:
len(df['family'].dropna()) #Not null
Out[117]:
In [17]:
null_data = df[df.isnull().any(axis=1)]
null_data.head()
Out[17]:
In [ ]:
#transform the database to consider only non null values
In [107]:
df2=df.dropna()
df2.info()
In [13]:
df.loc[df['pdbid'] == '1dy3']
Out[13]:
In [15]:
# Filling missing values
df3 = df.fillna('Unknown')
In [16]:
df3.loc[df3['pdbid'] == '1dy3']
Out[16]: