Pandas combining sparse columns in dataframe

Question

I am using Python, Pandas for data analysis. I have sparsely distributed data in different columns like following

| id | col1a | col1b | col2a | col2b | col3a         


        

Answer 1

Use:

import re

def fx(s):
    s = s.dropna()
    group = 'g' + re.search(r'\d+', s.index[0])[0]
    return pd.Series([group] + s.tolist(), index=['group', 'cola', 'colb'])

df1 = df.set_index('id').agg(fx, axis=1).reset_index()

---

# print(df1)

   id   group  cola   colb
0   1    g1    11.0   12.0
1   2    g2    21.0   86.0
2   3    g1    22.0   87.0
3   4    g3    545.0  32.0

Answer 2

Another alternative with pd.wide_to_long

m = pd.wide_to_long(df,['col'],'id','j',suffix='\d+\w+').reset_index()

(m.join(pd.DataFrame(m.pop('j').agg(list).tolist()))
  .assign(group=lambda x:x[0].radd('g'))
  .set_index(['id','group',1])['col'].unstack().dropna()
  .rename_axis(None,axis=1).add_prefix('col').reset_index())

---

   id group cola colb
0   1    g1   11   12
1   2    g2   21   86
2   3    g1   22   87
3   4    g3  545   32

Answer 3

This would a way of doing it:

df = pd.DataFrame({'id':[1,2,3,4],
                   'col1a':[11,np.nan,22,np.nan],
                   'col1b':[12,np.nan,87,np.nan],
                   'col2a':[np.nan,21,np.nan,np.nan],
                   'col2b':[np.nan,86,np.nan,np.nan],
                   'col3a':[np.nan,np.nan,np.nan,545],
                   'col3b':[np.nan,np.nan,np.nan,32]})
df_new = df.copy(deep=False)
df_new['group'] = 'g'+df_new['id'].astype(str)
df_new['cola'] = df_new[[x for x in df_new.columns if x.endswith('a')]].sum(axis=1)
df_new['colb'] = df_new[[x for x in df_new.columns if x.endswith('b')]].sum(axis=1)
df_new = df_new[['id','group','cola','colb']]
print(df_new)

Output:

  id group   cola  colb
0   1    g1   11.0  12.0
1   2    g2   21.0  86.0
2   3    g3   22.0  87.0
3   4    g4  545.0  32.0

So if you have more suffixes (colc, cold, cole, colf, etc...) you can create a loop and then use:

suffixes = ['a','b','c','d','e','f']
cols = ['id','group'] + ['col'+x for x in suffixes]
for i in suffixes:
   df_new['col'+i] = df_new[[x for x in df_new.columns if x.endswith(i)]].sum(axis=1)
df_new = df_new[cols]

Answer 4

Thanks to @CeliusStingher for providing the code for the dataframe :

One suggestion is to set the id as index, rearrange the columns, with the numbers extracted from the text. Create a multiIndex, and stack to get the final result :

#set id as index
df = df.set_index("id")

#pull out the numbers from each column
#so that you have (cola,1), (colb,1) ...
#add g to the numbers ... (cola, g1),(colb,g1), ...
#create a MultiIndex
#and reassign to the columns
df.columns = pd.MultiIndex.from_tuples([("".join((first,last)), f"g{second}")
                                        for first, second, last
                                        in df.columns.str.split("(\d)")],
                                       names=[None,"group"])

#stack the data 
#to get your result
df.stack()


                 cola   colb
    id  group       
    1   g1      11.0    12.0
    2   g2      21.0    86.0
    3   g1      22.0    87.0
    4   g3      545.0   32.0

Answer 5

You can use df.stack() assuming 'id' is your index else set 'id' as index. Then use pd.pivot_table.

df = df.stack().reset_index(name='val',level=1)
df['group'] = 'g'+ df['level_1'].str.extract('col(\d+)')
df['level_1'] = df['level_1'].str.replace('col(\d+)','')
df.pivot_table(index=['id','group'],columns='level_1',values='val')

level_1    cola  colb
id group
1  g1      11.0  12.0
2  g2      21.0  86.0
3  g1      22.0  87.0
4  g3     545.0  32.0