From b2b49cef6f98230cb36e1fd8fe3907a792e7a051 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Tue, 28 May 2024 18:03:52 +0000 Subject: [PATCH] split tensor docs (#4754) --- docs/developer.md | 4 +- docs/tensor.md | 192 ---------------------------------------- docs/tensor/creation.md | 23 +++++ docs/tensor/index.md | 34 +++++++ docs/tensor/movement.md | 25 ++++++ docs/tensor/ops.md | 108 ++++++++++++++++++++++ mkdocs.yml | 7 +- 7 files changed, 199 insertions(+), 194 deletions(-) create mode 100644 docs/tensor/creation.md create mode 100644 docs/tensor/index.md create mode 100644 docs/tensor/movement.md create mode 100644 docs/tensor/ops.md diff --git a/docs/developer.md b/docs/developer.md index 2503366a2d..f5b131816f 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -5,6 +5,8 @@ The tinygrad framework has four pieces * a lowering engine which converts ASTs into code that can run on the accelerator. * an execution engine which can run that code. +There is a good [bunch of tutorials](https://mesozoic-egg.github.io/tinygrad-notes/) by Di Zhu that go over tinygrad internals. + ## Frontend Everything in [Tensor](tensor.md) is syntactic sugar around [function.py](function.md), where the forwards and backwards passes are implemented for the different functions. There's about 25 of them, implemented using about 20 basic ops. Those basic ops go on to construct a graph of: @@ -41,4 +43,4 @@ Creating `ExecItem`, which has a run method options: members: true -Lists of `ExecItem` can be condensed into a single ExecItem with the Graph API (rename to Queue?) \ No newline at end of file +Lists of `ExecItem` can be condensed into a single ExecItem with the Graph API (rename to Queue?) diff --git a/docs/tensor.md b/docs/tensor.md index af159fe87d..8b13789179 100644 --- a/docs/tensor.md +++ b/docs/tensor.md @@ -1,193 +1 @@ -::: tinygrad.Tensor - options: - heading_level: 2 - members: false - show_source: false -## Properties - -::: tinygrad.Tensor.shape -::: tinygrad.Tensor.dtype -::: tinygrad.Tensor.device - -## Creation (basic) - -::: tinygrad.Tensor.empty -::: tinygrad.Tensor.zeros -::: tinygrad.Tensor.ones -::: tinygrad.Tensor.full -::: tinygrad.Tensor.arange -::: tinygrad.Tensor.eye -::: tinygrad.Tensor.full_like -::: tinygrad.Tensor.zeros_like -::: tinygrad.Tensor.ones_like - -## Creation (random) - -::: tinygrad.Tensor.rand -::: tinygrad.Tensor.randn -::: tinygrad.Tensor.randint -::: tinygrad.Tensor.normal -::: tinygrad.Tensor.uniform -::: tinygrad.Tensor.scaled_uniform -::: tinygrad.Tensor.glorot_uniform -::: tinygrad.Tensor.kaiming_uniform -::: tinygrad.Tensor.kaiming_normal - -## Data Access - -::: tinygrad.Tensor.data -::: tinygrad.Tensor.item -::: tinygrad.Tensor.tolist -::: tinygrad.Tensor.numpy - -## tinygrad ops - -::: tinygrad.Tensor.schedule_with_vars -::: tinygrad.Tensor.schedule -::: tinygrad.Tensor.realize -::: tinygrad.Tensor.replace -::: tinygrad.Tensor.assign -::: tinygrad.Tensor.detach -::: tinygrad.Tensor.to -::: tinygrad.Tensor.to_ -::: tinygrad.Tensor.shard -::: tinygrad.Tensor.shard_ -::: tinygrad.Tensor.contiguous -::: tinygrad.Tensor.contiguous_backward -::: tinygrad.Tensor.backward - -## Movement (low level) - -::: tinygrad.Tensor.view -::: tinygrad.Tensor.reshape -::: tinygrad.Tensor.expand -::: tinygrad.Tensor.permute -::: tinygrad.Tensor.flip -::: tinygrad.Tensor.shrink -::: tinygrad.Tensor.pad - -## Movement (high level) - -::: tinygrad.Tensor.gather -::: tinygrad.Tensor.cat -::: tinygrad.Tensor.stack -::: tinygrad.Tensor.repeat -::: tinygrad.Tensor.split -::: tinygrad.Tensor.chunk -::: tinygrad.Tensor.squeeze -::: tinygrad.Tensor.unsqueeze -::: tinygrad.Tensor.pad2d -::: tinygrad.Tensor.T -::: tinygrad.Tensor.transpose -::: tinygrad.Tensor.flatten -::: tinygrad.Tensor.unflatten - -## Reduce - -::: tinygrad.Tensor.sum -::: tinygrad.Tensor.max -::: tinygrad.Tensor.min -::: tinygrad.Tensor.mean -::: tinygrad.Tensor.var -::: tinygrad.Tensor.std -::: tinygrad.Tensor.softmax -::: tinygrad.Tensor.log_softmax -::: tinygrad.Tensor.logsumexp -::: tinygrad.Tensor.argmax -::: tinygrad.Tensor.argmin - -## Processing - -::: tinygrad.Tensor.avg_pool2d -::: tinygrad.Tensor.max_pool2d -::: tinygrad.Tensor.conv2d -::: tinygrad.Tensor.conv_transpose2d -::: tinygrad.Tensor.dot -::: tinygrad.Tensor.matmul -::: tinygrad.Tensor.einsum -::: tinygrad.Tensor.cumsum -::: tinygrad.Tensor.triu -::: tinygrad.Tensor.tril - -## Unary Ops (math) - -::: tinygrad.Tensor.logical_not -::: tinygrad.Tensor.neg -::: tinygrad.Tensor.log -::: tinygrad.Tensor.log2 -::: tinygrad.Tensor.exp -::: tinygrad.Tensor.exp2 -::: tinygrad.Tensor.sqrt -::: tinygrad.Tensor.rsqrt -::: tinygrad.Tensor.sin -::: tinygrad.Tensor.cos -::: tinygrad.Tensor.tan -::: tinygrad.Tensor.trunc -::: tinygrad.Tensor.ceil -::: tinygrad.Tensor.floor -::: tinygrad.Tensor.round -::: tinygrad.Tensor.lerp -::: tinygrad.Tensor.square -::: tinygrad.Tensor.clip -::: tinygrad.Tensor.sign -::: tinygrad.Tensor.abs -::: tinygrad.Tensor.reciprocal - -## Unary Ops (activation) - -::: tinygrad.Tensor.relu -::: tinygrad.Tensor.sigmoid -::: tinygrad.Tensor.elu -::: tinygrad.Tensor.celu -::: tinygrad.Tensor.swish -::: tinygrad.Tensor.silu -::: tinygrad.Tensor.relu6 -::: tinygrad.Tensor.hardswish -::: tinygrad.Tensor.tanh -::: tinygrad.Tensor.sinh -::: tinygrad.Tensor.cosh -::: tinygrad.Tensor.atanh -::: tinygrad.Tensor.asinh -::: tinygrad.Tensor.acosh -::: tinygrad.Tensor.hardtanh -::: tinygrad.Tensor.gelu -::: tinygrad.Tensor.quick_gelu -::: tinygrad.Tensor.leakyrelu -::: tinygrad.Tensor.mish -::: tinygrad.Tensor.softplus -::: tinygrad.Tensor.softsign - -## Elementwise Ops (broadcasted) - -::: tinygrad.Tensor.add -::: tinygrad.Tensor.sub -::: tinygrad.Tensor.mul -::: tinygrad.Tensor.div -::: tinygrad.Tensor.xor -::: tinygrad.Tensor.lshift -::: tinygrad.Tensor.rshift -::: tinygrad.Tensor.pow -::: tinygrad.Tensor.maximum -::: tinygrad.Tensor.minimum -::: tinygrad.Tensor.where - -## Neural Network Ops (functional) - -::: tinygrad.Tensor.linear -::: tinygrad.Tensor.sequential -::: tinygrad.Tensor.layernorm -::: tinygrad.Tensor.batchnorm -::: tinygrad.Tensor.dropout -::: tinygrad.Tensor.one_hot -::: tinygrad.Tensor.scaled_dot_product_attention -::: tinygrad.Tensor.binary_crossentropy -::: tinygrad.Tensor.binary_crossentropy_logits -::: tinygrad.Tensor.sparse_categorical_crossentropy - -## Casting Ops - -::: tinygrad.Tensor.cast -::: tinygrad.Tensor.bitcast -::: tinygrad.Tensor.float -::: tinygrad.Tensor.half diff --git a/docs/tensor/creation.md b/docs/tensor/creation.md new file mode 100644 index 0000000000..e3f2e5f4f4 --- /dev/null +++ b/docs/tensor/creation.md @@ -0,0 +1,23 @@ +## Creation (basic) + +::: tinygrad.Tensor.empty +::: tinygrad.Tensor.zeros +::: tinygrad.Tensor.ones +::: tinygrad.Tensor.full +::: tinygrad.Tensor.arange +::: tinygrad.Tensor.eye +::: tinygrad.Tensor.full_like +::: tinygrad.Tensor.zeros_like +::: tinygrad.Tensor.ones_like + +## Creation (random) + +::: tinygrad.Tensor.rand +::: tinygrad.Tensor.randn +::: tinygrad.Tensor.randint +::: tinygrad.Tensor.normal +::: tinygrad.Tensor.uniform +::: tinygrad.Tensor.scaled_uniform +::: tinygrad.Tensor.glorot_uniform +::: tinygrad.Tensor.kaiming_uniform +::: tinygrad.Tensor.kaiming_normal diff --git a/docs/tensor/index.md b/docs/tensor/index.md new file mode 100644 index 0000000000..28cdf3e38e --- /dev/null +++ b/docs/tensor/index.md @@ -0,0 +1,34 @@ +::: tinygrad.Tensor + options: + heading_level: 2 + members: false + show_source: false + +## Properties + +::: tinygrad.Tensor.shape +::: tinygrad.Tensor.dtype +::: tinygrad.Tensor.device + +## Data Access + +::: tinygrad.Tensor.data +::: tinygrad.Tensor.item +::: tinygrad.Tensor.tolist +::: tinygrad.Tensor.numpy + +## tinygrad ops + +::: tinygrad.Tensor.schedule_with_vars +::: tinygrad.Tensor.schedule +::: tinygrad.Tensor.realize +::: tinygrad.Tensor.replace +::: tinygrad.Tensor.assign +::: tinygrad.Tensor.detach +::: tinygrad.Tensor.to +::: tinygrad.Tensor.to_ +::: tinygrad.Tensor.shard +::: tinygrad.Tensor.shard_ +::: tinygrad.Tensor.contiguous +::: tinygrad.Tensor.contiguous_backward +::: tinygrad.Tensor.backward diff --git a/docs/tensor/movement.md b/docs/tensor/movement.md new file mode 100644 index 0000000000..ca9df8da61 --- /dev/null +++ b/docs/tensor/movement.md @@ -0,0 +1,25 @@ +## Movement (low level) + +::: tinygrad.Tensor.view +::: tinygrad.Tensor.reshape +::: tinygrad.Tensor.expand +::: tinygrad.Tensor.permute +::: tinygrad.Tensor.flip +::: tinygrad.Tensor.shrink +::: tinygrad.Tensor.pad + +## Movement (high level) + +::: tinygrad.Tensor.gather +::: tinygrad.Tensor.cat +::: tinygrad.Tensor.stack +::: tinygrad.Tensor.repeat +::: tinygrad.Tensor.split +::: tinygrad.Tensor.chunk +::: tinygrad.Tensor.squeeze +::: tinygrad.Tensor.unsqueeze +::: tinygrad.Tensor.pad2d +::: tinygrad.Tensor.T +::: tinygrad.Tensor.transpose +::: tinygrad.Tensor.flatten +::: tinygrad.Tensor.unflatten diff --git a/docs/tensor/ops.md b/docs/tensor/ops.md new file mode 100644 index 0000000000..c23a29727f --- /dev/null +++ b/docs/tensor/ops.md @@ -0,0 +1,108 @@ +## Reduce Ops + +::: tinygrad.Tensor.sum +::: tinygrad.Tensor.max +::: tinygrad.Tensor.min +::: tinygrad.Tensor.mean +::: tinygrad.Tensor.var +::: tinygrad.Tensor.std +::: tinygrad.Tensor.softmax +::: tinygrad.Tensor.log_softmax +::: tinygrad.Tensor.logsumexp +::: tinygrad.Tensor.argmax +::: tinygrad.Tensor.argmin + +## Processing Ops + +::: tinygrad.Tensor.avg_pool2d +::: tinygrad.Tensor.max_pool2d +::: tinygrad.Tensor.conv2d +::: tinygrad.Tensor.conv_transpose2d +::: tinygrad.Tensor.dot +::: tinygrad.Tensor.matmul +::: tinygrad.Tensor.einsum +::: tinygrad.Tensor.cumsum +::: tinygrad.Tensor.triu +::: tinygrad.Tensor.tril + +## Unary Ops (math) + +::: tinygrad.Tensor.logical_not +::: tinygrad.Tensor.neg +::: tinygrad.Tensor.log +::: tinygrad.Tensor.log2 +::: tinygrad.Tensor.exp +::: tinygrad.Tensor.exp2 +::: tinygrad.Tensor.sqrt +::: tinygrad.Tensor.rsqrt +::: tinygrad.Tensor.sin +::: tinygrad.Tensor.cos +::: tinygrad.Tensor.tan +::: tinygrad.Tensor.trunc +::: tinygrad.Tensor.ceil +::: tinygrad.Tensor.floor +::: tinygrad.Tensor.round +::: tinygrad.Tensor.lerp +::: tinygrad.Tensor.square +::: tinygrad.Tensor.clip +::: tinygrad.Tensor.sign +::: tinygrad.Tensor.abs +::: tinygrad.Tensor.reciprocal + +## Unary Ops (activation) + +::: tinygrad.Tensor.relu +::: tinygrad.Tensor.sigmoid +::: tinygrad.Tensor.elu +::: tinygrad.Tensor.celu +::: tinygrad.Tensor.swish +::: tinygrad.Tensor.silu +::: tinygrad.Tensor.relu6 +::: tinygrad.Tensor.hardswish +::: tinygrad.Tensor.tanh +::: tinygrad.Tensor.sinh +::: tinygrad.Tensor.cosh +::: tinygrad.Tensor.atanh +::: tinygrad.Tensor.asinh +::: tinygrad.Tensor.acosh +::: tinygrad.Tensor.hardtanh +::: tinygrad.Tensor.gelu +::: tinygrad.Tensor.quick_gelu +::: tinygrad.Tensor.leakyrelu +::: tinygrad.Tensor.mish +::: tinygrad.Tensor.softplus +::: tinygrad.Tensor.softsign + +## Elementwise Ops (broadcasted) + +::: tinygrad.Tensor.add +::: tinygrad.Tensor.sub +::: tinygrad.Tensor.mul +::: tinygrad.Tensor.div +::: tinygrad.Tensor.xor +::: tinygrad.Tensor.lshift +::: tinygrad.Tensor.rshift +::: tinygrad.Tensor.pow +::: tinygrad.Tensor.maximum +::: tinygrad.Tensor.minimum +::: tinygrad.Tensor.where + +## Neural Network Ops (functional) + +::: tinygrad.Tensor.linear +::: tinygrad.Tensor.sequential +::: tinygrad.Tensor.layernorm +::: tinygrad.Tensor.batchnorm +::: tinygrad.Tensor.dropout +::: tinygrad.Tensor.one_hot +::: tinygrad.Tensor.scaled_dot_product_attention +::: tinygrad.Tensor.binary_crossentropy +::: tinygrad.Tensor.binary_crossentropy_logits +::: tinygrad.Tensor.sparse_categorical_crossentropy + +## Casting Ops + +::: tinygrad.Tensor.cast +::: tinygrad.Tensor.bitcast +::: tinygrad.Tensor.float +::: tinygrad.Tensor.half diff --git a/mkdocs.yml b/mkdocs.yml index 9987d1640b..abbee85f79 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,7 +9,11 @@ nav: - Showcase: showcase.md - MNIST Tutorial: mnist.md - API Reference: - - Tensor: tensor.md + - Tensor: + - tensor/index.md + - Creation: tensor/creation.md + - Movement: tensor/movement.md + - Ops: tensor/ops.md - Function: function.md - dtypes: dtypes.md - nn (Neural Networks): nn.md @@ -65,6 +69,7 @@ theme: - navigation.sections - navigation.expand - navigation.top + - navigation.path - search.highlight - search.suggest - toc.follow