from copy import copy
from statistics import mode, median, median_low, median_high, mean, stdev
from visidata import vd, Column, ColumnAttr, vlen, RowColorizer, asyncthread, Progress, wrapply, TypedExceptionWrapper
from visidata import BaseSheet, TableSheet, ColumnsSheet, IndexSheet
vd.option('describe_aggrs', 'mean stdev', 'numeric aggregators to calculate on Describe sheet', help=vd.help_aggregators)
[docs]
@Column.api
def isError(col, row):
'Return True if the computed or typed value for *row* in this column is an error.'
try:
v = col.getValue(row)
if v is not None:
col.type(v)
return False
except Exception:
return True
class DescribeColumn(Column):
def __init__(self, name, **kwargs):
kwargs.setdefault('width', 10)
super().__init__(name, getter=lambda col,srccol: col.sheet.describeData[srccol].get(col.expr, ''), expr=name, **kwargs)
# rowdef: Column from source sheet
class DescribeSheet(ColumnsSheet):
# rowtype = 'columns'
guide = '''
# Describe Sheet
This `Describe Sheet` shows a few basic metrics over data in {sheet.displaySource}, with each column represented by a row.
For example, row {sheet.cursorRowIndex} describes the _{sheet.cursorRow.name}_ column, showing its minimum value, maximum value, mean, median, and other measures.
'''
precious = True
columns = [
ColumnAttr('sheet', 'sheet', width=0),
ColumnAttr('column', 'name'),
ColumnAttr('type', 'typestr', width=0),
DescribeColumn('errors', type=vlen),
DescribeColumn('nulls', type=vlen),
DescribeColumn('distinct',type=vlen),
DescribeColumn('mode', type=str),
DescribeColumn('min', type=str),
DescribeColumn('max', type=str),
DescribeColumn('sum'),
DescribeColumn('median', type=str),
]
colorizers = [
RowColorizer(7, 'color_key_col', lambda s,c,r,v: r and r in r.sheet.keyCols),
]
nKeys = 2
def loader(self):
super().loader()
self.rows = [c for c in self.rows if not c.hidden]
self.describeData = { col: {} for col in self.rows }
self.resetCols()
for aggrname in vd.options.describe_aggrs.split():
self.addColumn(DescribeColumn(aggrname, type=float))
for srccol in Progress(self.rows, 'categorizing'):
if not srccol.hidden:
self.reloadColumn(srccol)
def reloadColumn(self, srccol):
d = self.describeData[srccol]
isNull = srccol.sheet.isNullFunc()
vals = list()
d['errors'] = list()
d['nulls'] = list()
d['distinct'] = set()
for sr in Progress(srccol.sheet.rows, 'calculating'):
try:
v = srccol.getValue(sr)
if isNull(v):
d['nulls'].append(sr)
v = None # all nulls count as one distinct value
else:
v = srccol.type(v)
vals.append(v)
d['distinct'].add(v)
except Exception:
d['errors'].append(sr)
d['mode'] = self.calcStatistic(d, mode, vals)
if vd.isNumeric(srccol):
for func in [min, max, sum, median]: # use type
d[func.__name__] = self.calcStatistic(d, func, vals)
for aggrname in vd.options.describe_aggrs.split():
aggr = vd.aggregators[aggrname].funcValues
d[aggrname] = self.calcStatistic(d, aggr, vals)
def calcStatistic(self, d, func, *args, **kwargs):
r = wrapply(func, *args, **kwargs)
d[func.__name__] = r
return r
def describedRows(self, col, srccol):
'Return rows from the source sheet of *srccol* which are described by the statistic in *col*.'
if not isinstance(col, DescribeColumn):
vd.fail('no rows described in this column')
d = self.describeData[srccol]
statname = col.expr
if statname in ('errors', 'nulls'):
return list(d[statname])
stat = d.get(statname)
if stat is None or isinstance(stat, TypedExceptionWrapper):
vd.fail(f'no {statname} for this column')
isNull = srccol.sheet.isNullFunc()
ret = []
medianRows = []
for sr in Progress(srccol.sheet.rows, 'gathering'):
try:
v = srccol.getValue(sr)
if isNull(v):
if statname == 'distinct':
ret.append(sr)
continue
v = srccol.type(v)
except Exception:
continue
# match typed value, as computed in reloadColumn #3223
if statname == 'median':
medianRows.append((v, sr))
elif statname in ('mode', 'min', 'max'):
if v == stat:
ret.append(sr)
else: # distinct and aggregators: all typed rows
ret.append(sr)
if medianRows:
vals = [v for v, sr in medianRows]
middle = (median_low(vals), median_high(vals))
if middle[0] != middle[1]:
vd.status(f'median {stat} is between two values; using rows with {middle[0]} or {middle[1]}')
ret = [sr for v, sr in medianRows if v in middle]
return ret
def openCell(self, col, row):
'open copy of source sheet with rows described in current cell'
vs=copy(row.sheet)
vs.rows=self.describedRows(col, row)
vs.name+="_%s_%s"%(row.name,col.name)
return vs
TableSheet.addCommand('I', 'describe-sheet', 'vd.push(DescribeSheet(sheet.name+"_describe", source=[sheet]))', 'open Describe Sheet with descriptive statistics for all visible columns')
BaseSheet.addCommand('gI', 'describe-all', 'vd.push(DescribeSheet("describe_all", source=vd.stackedSheets))', 'open Describe Sheet with description statistics for all visible columns from all sheets')
IndexSheet.addCommand('gI', 'describe-selected', 'vd.push(DescribeSheet("describe_all", source=selectedRows))', 'open Describe Sheet with all visible columns from selected sheets')
DescribeSheet.addCommand('zs', 'select-cell', 'cursorRow.sheet.select(describedRows(cursorCol, cursorRow))', 'select rows on source sheet which are being described in current cell')
DescribeSheet.addCommand('zu', 'unselect-cell', 'cursorRow.sheet.unselect(describedRows(cursorCol, cursorRow))', 'unselect rows on source sheet which are being described in current cell')
vd.addMenuItems('Data > Statistics > describe-sheet')
vd.addGlobals({'DescribeSheet':DescribeSheet})