Time Series 機械学習のための Python 必須ライブラリ（Pandas 3）

!cat ex1.csv

Out [2]:

,a,b,c,d
0,1,2,3,4
1,5,6,7,8
2,9,10,11,12
3,13,14,15,16

In [3]:

df = pd.read_csv('ex1.csv', index_col = 0)

In [4]:

df

Out [5]:

	a	b	c	d
0	1	2	3	4
1	5	6	7	8
2	9	10	11	12
3	13	14	15	16

In [6]:

df_1.drop(['Unnamed: 0'], axis = 1, inplace = True)

In [7]:

df_1

Out [8]:

	a	b	c	d
0	1	2	3	4
1	5	6	7	8
2	9	10	11	12
3	13	14	15	16

In [9]:

df = pd.read_csv('ex1.csv', index_col = 0)

In [10]:

df

Out [11]:

	a	b	c	d
0	1	2	3	4
1	5	6	7	8
2	9	10	11	12
3	13	14	15	16

In [12]:

df_1.drop(['Unnamed: 0'], axis = 1, inplace = True)

In [13]:

df_1

Out [14]:

	a	b	c	d
0	1	2	3	4
1	5	6	7	8
2	9	10	11	12
3	13	14	15	16

chunksize = 10 **3
for cnt, chunk in enumerate(pd.read_csv(filename, chunksize = chunksize)):
    do_something(chunk)

for filename in os.listdir(dir):
    if filename.endswith('.csv'):
        df = pd.read_csv(dir + '/' + filename)

In [15]:

series = pd.Series([2,4,6,np.nan,10,np.nan])

In [16]:

series

Out [17]:

0     2.0
1     4.0
2     6.0
3     NaN
4    10.0
5     NaN
dtype: float64

In [18]:

series.fillna(1)

Out [19]:

0     2.0
1     4.0
2     6.0
3     1.0
4    10.0
5     1.0
dtype: float64

In [20]:

series.isnull()

Out [21]:

0    False
1    False
2    False
3     True
4    False
5     True
dtype: bool

In [22]:

series.isnull().sum()

Out [23]:

2

In [24]:

np.where(series.isnull())

Out [25]:

(array([3, 5]),)

In [26]:

df = pd.DataFrame(np.arange(12).reshape(6,2), index=[1,2,2,3,3,4], columns=['a','b'])

In [27]:

df

Out [28]:

	a	b
1	0	1
2	2	3
2	4	5
3	6	7
3	8	9
4	10	11

In [29]:

df.iat[5,0] = 0
df.iat[5,1] = 1

In [30]:

df

Out [31]:

	a	b
1	0	1
2	2	3
2	4	5
3	6	7
3	8	9
4	0	1

In [32]:

df.duplicated()

Out [33]:

1    False
2    False
2    False
3    False
3    False
4     True
dtype: bool

さらに、keep = ‘last’と指定すると、後で重複するデータではなく、前に位置にtrueを与えることもできます。

In [34]:

df.duplicated(keep = 'last')

Out [35]:

1     True
2    False
2    False
3    False
3    False
4    False
dtype: bool

df.drop_duplicates()

Out [37]:

	a	b
1	0	1
2	2	3
2	4	5
3	6	7
3	8	9

In [38]:

df

Out [39]:

	a	b
1	0	1
2	2	3
2	4	5
3	6	7
3	8	9
4	10	1

In [40]:

df.duplicated('b')

Out [41]:

1    False
2    False
2    False
3    False
3    False
4     True
dtype: bool

In [42]:

df.drop_duplicates('b')

Out [43]:

	a	b
1	0	1
2	2	3
2	4	5
3	6	7
3	8	9

In [44]:

df = pd.DataFrame([[70, 80, 50],[30, str(30), 40],[76,43,54],[55,75,95]],
                  columns = ['国語」、「英語」、「数学'])

In [45]:

df

Out [46]:

	国語    英語   数学
0	70	80	50
1	30	30	40
2	76	43	54
3	55	75	95

In [47]:

df['英語'].str.contains('0')

Out [48]:

0     NaN
1    True
2     NaN
3     NaN
Name: 英語, dtype: object

In [49]:

df = pd.DataFrame([[70, 80, 50],[30, str(30), 40],[76,43,54],[55,75,95]],
                  columns = ['国語」、「英語」、「数学'])

In [50]:

df

Out [51]:

       国語    英語    数学
0	70	80	50
1	30	30	40
2	76	43	54
3	55	75	95

In [52]:

df['英語'].str.contains('0')

Out [53]:

0     NaN
1    True
2     NaN
3     NaN
Name: 英語, dtype: object

In [54]:

df = pd.DataFrame([[70, 80, 50],[30, np.NAN, 40],[76,43,54],[55,75,95]],
                  columns = ['国語」、「英語」、「数学'])

In [55]:

df

Out [56]:

	国語    英語   数学
0	70	80.0	50
1	30	NaN	40
2	76	43.0	54
3	55	75.0	95

In [57]:

df.replace([np.NaN, 30], 0)

Out [58]:

	国語    英語   数学
0	70	80.0	50
1	0	0.0	40
2	76	43.0	54
3	55	75.0	95

In [59]:

df1 = pd.DataFrame({'a':['a0','a1','a2','a3'],
                   'b':['b0','b1','b2','b3'],
                   'c':['c0','c1','c2','c3']},
                  index = [0,1,2,3])

df2 = pd.DataFrame({'a':['a2','a3','a4','a5'],
                   'b':['b2','b3','b4','b5'],
                   'c':['c2','c3','c4','c5'],
                   'd':['d2','d3','d4','d5']},
                   index = [2,3,4,5])

In [60]:

df1

Out [61]:

	a	b	c
0	a0	b0	c0
1	a1	b1	c1
2	a2	b2	c2
3	a3	b3	c3

In [62]:

df2

Out [63]:

	a	b	c	d
2	a2	b2	c2	d2
3	a3	b3	c3	d3
4	a4	b4	c4	d4
5	a5	b5	c5	d5

In [64]:

pd.concat([df1,df2])

Out [65]:

	a	b	c	d
0	a0	b0	c0	NaN
1	a1	b1	c1	NaN
2	a2	b2	c2	NaN
3	a3	b3	c3	NaN
2	a2	b2	c2	d2
3	a3	b3	c3	d3
4	a4	b4	c4	d4
5	a5	b5	c5	d5

通常のpd.concatを使って合わせるとインデックスもそのままついてしまい、オプションでignore_index = Tureを与えればreset_indexになったインデックスを使うことができる。

In [66]:

pd.concat([df1,df2], ignore_index=True)

Out [67]:

	a	b	c	d
0	a0	b0	c0	NaN
1	a1	b1	c1	NaN
2	a2	b2	c2	NaN
3	a3	b3	c3	NaN
4	a2	b2	c2	d2
5	a3	b3	c3	d3
6	a4	b4	c4	d4
7	a5	b5	c5	d5

In [68]:

pd.concat([df1,df2], join= 'inner')

Out [69]:

	a	b	c
0	a0	b0	c0
1	a1	b1	c1
2	a2	b2	c2
3	a3	b3	c3
2	a2	b2	c2
3	a3	b3	c3
4	a4	b4	c4
5	a5	b5	c5

df1 = pd.DataFrame(np.arange(10).reshape(5,2), columns = ['a','b'])
df2 = pd.DataFrame(np.arange(12).reshape(6,2), columns = ['a','c'])

df1

Out [72]:

	a	b
0	0	1
1	2	3
2	4	5
3	6	7
4	8	9

In [73]:

df2

Out [74]:

	c	d
0	0	1
1	2	3
2	4	5
3	6	7
4	8	9
5	10	11

In [75]:

pd.merge(df1,df2, left_on ='a', right_on='c', how = 'inner')

Out [76]:

	a	b	c	d
0	0	1	0	1
1	2	3	2	3
2	4	5	4	5
3	6	7	6	7
4	8	9	8	9

In [77]:

df1 = pd.DataFrame(np.arange(10).reshape(5,2), columns = ['a','b'])
df2 = pd.DataFrame(np.arange(12).reshape(6,2), columns = ['a','c'])
df3 = pd.DataFrame(np.arange(14).reshape(7,2), columns = ['a','d'])
df_list = [df1,df2,df3]

In [78]:

from functools import reduce
reduce(lambda left, right: pd.merge(left, right, on = ['a'], how = 'outer'), df_list)

Out [79]:

	a	b	c	d
0	0	1.0	1.0	1
1	2	3.0	3.0	3
2	4	5.0	5.0	5
3	6	7.0	7.0	7
4	8	9.0	9.0	9
5	10	NaN	11.0	11
6	12	NaN	NaN	13

In [80]:

df = pd.DataFrame({'id': [1, 1, 2, 2, 3, 3], 
                   'col': ['x1', 'x2', 'x1', 'x2', 'x1', 'x2'], 
                   'num': [30, 10, 70, 40, 20, 80], 
                   'str': ['a', 'a', 'b', 'b', 'c', 'c']})

In [81]:

df

Out [82]:

	id	col	num	str
0	1	x1	30	a
1	1	x2	10	a
2	2	x1	70	b
3	2	x2	40	b
4	3	x1	20	c
5	3	x2	80	c

In [83]:

df.pivot('id','col','num')

Out [84]:

col	x1	x2
id		
1	a	a
2	b	b
3	c	c

In [85]:

pd.pivot_table(df, index = 'id', columns = 'col', values = 'str', aggfunc = 'first')

Out [86]:

col	x1	x2
id		
1	a	a
2	b	b
3	c	c

글 | AI 1팀 김기중