eff63a8c-2f7e-4fc5-97ce-7f600dae0bc7.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import copy
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path

os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
import torch
torch.empty(1, device="cuda", requires_grad=True).backward() # prevents a bug on some systems
from torch import Tensor, nn
import torch.nn.functional as F
import torch.distributed as dist
# use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention
#torch._inductor.config.coordinate_descent_tuning = True # turn this on for a slightly faster run (but much slower compile time)

# -----------------------------------------------------------------------------
# Custom operators : FP8 matmul by @YouJiacheng

@torch.library.custom_op("nanogpt::mm", mutates_args=())
def mm_op(x: Tensor, w: Tensor, x_s: float, w_s: float, grad_s: float) -> tuple[Tensor, Tensor, Tensor]:
    @torch.compile
    def impl(x: Tensor, w: Tensor):
        assert x.is_contiguous() and w.is_contiguous()
        x_f8 = x.mul(x_s).to(torch.float8_e4m3fn)
        w_f8 = w.mul(w_s).to(torch.float8_e4m3fn)
        out = torch._scaled_mm(
            x_f8,
            w_f8.t(),
            out_dtype=torch.bfloat16,
            scale_a=x.new_tensor(1 / x_s, dtype=torch.float32),
            scale_b=x.new_tensor(1 / w_s, dtype=torch.float32),
            use_fast_accum=True,
        )
        return out, x_f8, w_f8

    return impl(x, w)

@mm_op.register_fake
def _(x: Tensor, w: Tensor, *_):
    assert x.ndim == w.ndim == 2
    assert x.shape[1] == w.shape[1]
    assert x.device == w.device
    assert x.is_contiguous() and w.is_contiguous()
    return x @ w.t(), x.to(torch.float8_e4m3fn), w.to(torch.float8_e4m3fn)

@torch.library.custom_op("nanogpt::mm_backward", mutates_args=())
def mm_backward_op(g: Tensor, x_f8: Tensor, w_f8: Tensor, x_s: float, w_s: float, grad_s: float) -> tuple[Tensor, Tensor]:
    @torch.compile
    def impl(grad: Tensor, x_f8: Tensor, w_f8: Tensor):
        assert grad.is_contiguous()
        x_inv_s = grad.new_tensor(1 / x_s, dtype=torch.float32)
        w_inv_s = grad.new_tensor(1 / w_s, dtype=torch.float32)
        grad_inv_s = grad.new_tensor(1 / grad_s, dtype=torch.float32)
        grad_f8 = grad.mul(grad_s).to(torch.float8_e5m2)
        grad_x = torch._scaled_mm(
            grad_f8,
            w_f8.t().contiguous().t(),
            out_dtype=torch.bfloat16,
            scale_a=grad_inv_s,
            scale_b=w_inv_s,
            use_fast_accum=False,
        )
        # faster than grad_f8_t @ x_f8, for (d_out, d_in) == (50304, 768)
        grad_w = torch._scaled_mm(
            x_f8.t().contiguous(),
            grad_f8.t().contiguous().t(),
            out_dtype=torch.float32,
            scale_a=x_inv_s,
            scale_b=grad_inv_s,
            use_fast_accum=False,
        ).t()
        return grad_x, grad_w

    return impl(g, x_f8, w_f8)

@mm_backward_op.register_fake
def _(g: Tensor, x_f8: Tensor, w_f8: Tensor, *_):
    return x_f8.to(torch.bfloat16), w_f8.to(torch.float32)

def backward(ctx, grad_out: Tensor, *_):
    x_f8, w_f8 = ctx.saved_tensors
    x_s, w_s, grad_s = ctx.scales
    grad_x, grad_w = torch.ops.nanogpt.mm_backward(
        grad_out, x_f8, w_f8, x_s, w_s, grad_s
    )
    return grad_x, grad_w, None, None, None

def setup_context(ctx: torch.autograd.function.FunctionCtx, inputs, output):
    *_, x_s, w_s, grad_s = inputs
    _, x_f8, w_f8 = output
    ctx.save_for_backward(x_f8, w_f8)
    ctx.scales = x_s, w_s, grad_s
    ctx.set_materialize_grads(False)

mm_op.register_autograd(backward, setup_context=setup_context)

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G: Tensor, steps: int) -> Tensor:
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert G.ndim >= 2 # batched Muon implementation by @scottjmaddox, and put into practice in the record by @YouJiacheng
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    if G.size(-2) > G.size(-1):
        X = X.mT

    # Ensure spectral norm is at most 1
    X = X / (X.norm(dim=(-2, -1), keepdim=True) + 1e-7)
    # Perform the NS iterations
    for _ in range(steps):
        A = X @ X.mT
        B = b * A + c * A @ A # quintic computation strategy adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    
    if G.size(-2) > G.size(-1):
        X = X.mT
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven"t tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5, rank=0, world_size=1):
        self.rank = rank
        self.world_size = world_size
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params: list[Tensor] = [*params]
        assert all(isinstance(p, Tensor) for p in params)
        sizes = {p.numel() for p in params}
        def create_update_buffer(size: int):
            b = torch.empty(self.world_size, size, dtype=torch.bfloat16, device="cuda")
            return dict(update_buffer=b, update_buffer_views=[b[i] for i in range(self.world_size)])
        param_groups = [
            dict(params=[p for p in params if p.numel() == size], **create_update_buffer(size)) for size in sizes]
        super().__init__(param_groups, defaults)

    @torch.no_grad()
    def step(self):
        for group in self.param_groups:
            lr = group["lr"]
            momentum = group["momentum"]
            nesterov = group["nesterov"]
            ns_steps = group["ns_steps"]
            update_buffer = group["update_buffer"]
            update_buffer_views: list[Tensor] = group["update_buffer_views"]
            # generate weight updates in distributed fashion
            params: list[Tensor] = group["params"]
            handle = None
            params_world = None
            def update_prev(): # optimized Muon implementation contributed by @YouJiacheng
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffer_views):
                    p_world.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(-2) / p_world.size(-1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                if base_i + self.rank < len(params):
                    p = params[base_i + self.rank]
                    g = p.grad
                    assert g is not None
                    state = self.state[p]
                    if "momentum_buffer" not in state:
                        state["momentum_buffer"] = torch.zeros_like(g)
                    buf: Tensor = state["momentum_buffer"]
                    buf.lerp_(g, 1 - momentum)
                    g = g.lerp_(buf, momentum) if nesterov else buf
                    g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                else:
                    g = update_buffer_views[self.rank]
                update_prev() # async all_gather instead of sync all_reduce by @YouJiacheng
                handle = dist.all_gather_into_tensor(update_buffer, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the model

def norm(x: Tensor):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):
    def __init__(self, in_features: int, out_features: int, use_fp8: bool = False, x_s: float = 1.0, w_s: float = 1.0, grad_s: float = 1.0):
        super().__init__(in_features, out_features, bias=False)
        self.use_fp8 = use_fp8
        self.x_s = x_s
        self.w_s = w_s
        self.grad_s = grad_s

    def reset_parameters(self) -> None:
        std = 0.5 * (self.in_features ** -0.5) # 0.5 is a bit better than the default 1/sqrt(3)
        bound = (3 ** 0.5) * std
        with torch.no_grad():
            self.weight.uniform_(-bound, bound)

    def forward(self, x: Tensor):
        if self.use_fp8 and self.training:
            _x = x.flatten(0, -2)
            out: Tensor = torch.ops.nanogpt.mm(_x, self.weight, x_s=self.x_s, w_s=self.w_s, grad_s=self.grad_s)[0]
            return out.reshape(*x.shape[:-1], -1)
        else:
            return F.linear(x, self.weight.type_as(x))

class Rotary(nn.Module):
    def __init__(self, dim: int, max_seq_len: int):
        super().__init__()
        # half-truncate RoPE by @YouJiacheng (w/ base freq tuning)
        angular_freq = (1 / 1024) ** torch.linspace(0, 1, steps=dim//4, dtype=torch.float32)
        angular_freq = torch.cat([angular_freq, angular_freq.new_zeros(dim//4)])
        t = torch.arange(max_seq_len, dtype=torch.float32)
        theta = torch.einsum("i,j -> ij", t, angular_freq)
        self.cos = nn.Buffer(theta.cos(), persistent=False)
        self.sin = nn.Buffer(theta.sin(), persistent=False)

    def forward(self, x_BTHD: Tensor):
        assert self.cos.size(0) >= x_BTHD.size(-3)
        cos, sin = self.cos[None, :x_BTHD.size(-3), None, :], self.sin[None, :x_BTHD.size(-3), None, :]
        x1, x2 = x_BTHD.to(dtype=torch.float32).chunk(2, dim=-1)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x_BTHD)

class CausalSelfAttention(nn.Module):
    def __init__(self, dim: int, num_heads: int, max_seq_len: int, head_dim=128):
        super().__init__()
        self.num_heads = num_heads
        self.head_dim = head_dim
        hdim = num_heads * head_dim
        std = 0.5 * (dim ** -0.5)
        bound = (3 ** 0.5) * std # improved init scale by @YouJiacheng
        # merged QKV weights: suggested by many, implemented by @fernbear.bsky.social, and further improved by @YouJiacheng
        # https://x.com/hi_tysam/status/1879699187107033311
        self.qkv_w = nn.Parameter(torch.empty(3, hdim, dim).uniform_(-bound, bound))
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(head_dim, max_seq_len)
        self.c_proj = CastedLinear(hdim, dim)
        self.c_proj.weight.detach().zero_() # zero init suggested by @Grad62304977
        # scale the attention logits by given constant, instead of the default head_dim**-0.5, by @leloykun
        # inspired by learnable scalars used by @brendanh0gan https://x.com/hi_tysam/status/1879693583898591283
        self.attn_scale = 0.12

    def forward(self, x: Tensor, ve: Tensor | None, block_mask: BlockMask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q, k, v = F.linear(x, self.qkv_w.flatten(end_dim=1).type_as(x)).view(B, T, 3 * self.num_heads, self.head_dim).chunk(3, dim=-2)
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        if ve is not None:
            v = self.lambdas[0] * v + self.lambdas[1] * ve.view_as(v) # @KoszarskyB & @Grad62304977
        else: # skip mid-layers token value embeddings by @YouJiacheng
            v = self.lambdas[0] * v
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, scale=self.attn_scale).transpose(1, 2)
        y = y.contiguous().view(B, T, self.num_heads * self.head_dim) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):
    def __init__(self, dim: int):
        super().__init__()
        hdim = 4 * dim
        self.c_fc = CastedLinear(dim, hdim)
        self.c_proj = CastedLinear(hdim, dim)
        self.c_proj.weight.detach().zero_() # zero init suggested by @Grad62304977

    def forward(self, x: Tensor):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):
    def __init__(self, dim: int, num_heads: int, max_seq_len: int, layer_idx: int):
        super().__init__()
        # skip attention of blocks.7 (the 8th layer) by @YouJiacheng
        self.attn = CausalSelfAttention(dim, num_heads, max_seq_len) if layer_idx != 7 else None
        self.mlp = MLP(dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: Tensor, ve: Tensor | None, x0: Tensor, block_mask: BlockMask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        if self.attn is not None:
            x = x + self.attn(norm(x), ve, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main model

def next_multiple_of_n(v: float | int, *, n: int):
    return next(x for x in range(n, int(v) + 1 + n, n) if x >= v)

class GPT(nn.Module):
    def __init__(self, vocab_size: int, num_layers: int, num_heads: int, model_dim: int, max_seq_len: int):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, model_dim)
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual implementation following https://arxiv.org/abs/2410.17897
        # value embedding code simplification inspired by @ragulpr https://github.com/KellerJordan/modded-nanogpt/pull/78
        self.value_embeds = nn.ModuleList([nn.Embedding(vocab_size, model_dim) for _ in range(3)])
        self.blocks = nn.ModuleList([Block(model_dim, num_heads, max_seq_len, i) for i in range(num_layers)])
        # there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency.
        # suggested to me by @Grad62304977. this originates from Karpathy's experiments.
        self.lm_head = CastedLinear(model_dim, next_multiple_of_n(vocab_size, n=128), use_fp8=True, x_s=2.0, w_s=2.0**9, grad_s=2.0**19)
        self.lm_head.weight.detach().zero_() # @Grad62304977
        # Add learnable skip connection weights for decoder layers
        assert num_layers % 2 == 0
        self.skip_weights = nn.Parameter(torch.ones(num_layers//2))

    def create_blockmasks(self, input_seq: Tensor, sliding_window_num_blocks: Tensor):
        BLOCK_SIZE = 128
        docs = (input_seq == 50256).cumsum(0)

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_blockmask: Tensor):
            num_blocks = dense_blockmask.sum(dim=-1, dtype=torch.int32)
            indices = dense_blockmask.argsort(dim=-1, descending=False, stable=True).flip(-1).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        # manual block mask creation by @YouJiacheng
        assert len(input_seq) % BLOCK_SIZE == 0
        NUM_BLOCKS = len(input_seq) // BLOCK_SIZE
        block_idx = torch.arange(NUM_BLOCKS, dtype=torch.int32, device="cuda")
        causal_blockmask_any = block_idx[:, None] >= block_idx
        causal_blockmask_all = block_idx[:, None] > block_idx
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()
        document_blockmask_any = (docs_low[:, None] <= docs_high) & (docs_high[:, None] >= docs_low)
        document_blockmask_all = (docs_low[:, None] == docs_high) & (docs_high[:, None] == docs_low)
        blockmask_any = causal_blockmask_any & document_blockmask_any
        blockmask_all = causal_blockmask_all & document_blockmask_all
        partial_kv_num_blocks, partial_kv_indices = dense_to_ordered(blockmask_any & ~blockmask_all)
        full_kv_num_blocks, full_kv_indices = dense_to_ordered(blockmask_all)
        def build_bm(window_size_blocks: Tensor) -> BlockMask:
            return BlockMask.from_kv_blocks(
                torch.clamp_max(partial_kv_num_blocks, torch.clamp_min(window_size_blocks - full_kv_num_blocks, 1)),
                partial_kv_indices,
                torch.clamp_max(full_kv_num_blocks, window_size_blocks - 1),
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )
        # Long-short SWA block masks by @leloykun & @YouJiacheng, adapated from suggestion by @Grad62304977, following Gemma 2 paper
        return build_bm(sliding_window_num_blocks), build_bm(sliding_window_num_blocks // 2)

    def forward(self, input_seq: Tensor, target_seq: Tensor, sliding_window_num_blocks: Tensor):
        assert input_seq.ndim == 1

        ve = [value_embed(input_seq) for value_embed in self.value_embeds]
        # 012 ... 012 structure on token value embeddings by @YouJiacheng, improved on @leloykun's U-net structure
        ve = [ve[0], ve[1], ve[2]] + [None] * (len(self.blocks) - 6) + [ve[0], ve[1], ve[2]]
        assert len(ve) == len(self.blocks)

        long_bm, short_bm = self.create_blockmasks(input_seq, sliding_window_num_blocks)
        block_masks = [long_bm, short_bm, short_bm, short_bm, long_bm, short_bm, short_bm, long_bm, short_bm, short_bm, short_bm, long_bm]
        assert len(block_masks) == len(self.blocks)

        x = x0 = norm(self.embed(input_seq)[None]) # use of norm here by @Grad62304977

        # U-net design by @brendanh0gan
        skip_connections = []
        n = len(self.skip_weights)
        for i in range(len(self.blocks)):
            if i >= n:
                x = x + self.skip_weights[i - n] * skip_connections.pop()
            x = self.blocks[i](x, ve[i], x0, block_masks[i])
            if i < n:
                skip_connections.append(x)

        x = norm(x)
        logits = self.lm_head(x)
        # @Grad62304977 added tanh softcapping following Gemma 2 paper, @KoszarskyB reduced it from 30 to 15, @YouJiacheng shifted it by +15 (2*sigmoid(2*x)=tanh(x)+1)
        logits = 30 * torch.sigmoid(logits.float() / 7.5)
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target_seq)
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _load_data_shard(file: Path):
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32) # header is 256 int32
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    num_tokens = int(header[2]) # number of tokens (claimed)
    with file.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True) # avoid pin_memory copy by @YouJiacheng
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy()) # avoid bytes->array copy by @YouJiacheng
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header"
    return tokens

def distributed_data_generator(filename_pattern: str, batch_size: int, rank : int, world_size : int):
    files = sorted(Path.cwd().glob(filename_pattern))
    assert batch_size % world_size == 0
    local_batch_size = batch_size // world_size
    file_iter = iter(files) # use itertools.cycle(files) instead if you want to do multi-epoch training
    tokens, pos = _load_data_shard(next(file_iter)), 0
    while True:
        if pos + batch_size + 1 >= len(tokens):
            tokens, pos = _load_data_shard(next(file_iter)), 0
        buf = tokens[pos + rank * local_batch_size:][:local_batch_size + 1]
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # no sync on host side;
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # H2D in another stream isn"t helpful.
        pos += batch_size
        yield inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data
    train_files = "data/fineweb10B/fineweb_train_*.bin" # input .bin to train on
    val_files = "data/fineweb10B/fineweb_val_*.bin" # input .bin to eval validation loss on
    val_tokens = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    # optimization
    num_iterations = 1770 # number of iterations to run
    cooldown_frac = 0.4 # fraction of training spent cooling down the learning rate
    # evaluation and logging
    val_loss_every = 125 # every how many steps to evaluate val loss? 0 for only at the end
    # implementation
    seq_len = 48*1024 # FlexAttention sequence length
    val_seq_len = 4*64*1024 # FlexAttention sequence length for validation
    save_checkpoint = False
args = Hyperparameters()

# torchrun sets these env variables
rank = int(os.environ["RANK"])
world_size = int(os.environ["WORLD_SIZE"])
assert world_size == 8 # this code is designed for 8xH100
assert torch.cuda.is_available()
device = torch.device("cuda", int(os.environ["LOCAL_RANK"]))
torch.cuda.set_device(device)
dist.init_process_group(backend="nccl", device_id=device)
dist.barrier()
master_process = (rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    os.makedirs("logs", exist_ok=True)
    logfile = f"logs/{run_id}.txt"
    print(logfile)
def print0(s, console=False):
    if master_process:
        with open(logfile, "a") as f:
            if console:
                print(s)
            print(s, file=f)

# begin by printing this file (the Python code)
print0(code)
print0("="*100)
# log information about the hardware/software environment this is running on
print0(f"Running Python {sys.version}")
print0(f"Running PyTorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}")
def nvidia_smi():
    import subprocess  # avoid top level import
    return subprocess.run(["nvidia-smi"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True).stdout
print0(nvidia_smi())
print0("="*100)

model: nn.Module = GPT(vocab_size=50257, num_layers=12, num_heads=6, model_dim=768, max_seq_len=max(args.seq_len, args.val_seq_len)).cuda()
for m in model.modules():
    if isinstance(m, nn.Embedding):
        m.bfloat16()
for param in model.parameters():
    dist.broadcast(param.detach(), 0)

# collect the parameters to optimize
hidden_matrix_params = [p for n, p in model.blocks.named_parameters() if p.ndim >= 2 and "embed" not in n]
embed_params = [p for n, p in model.named_parameters() if "embed" in n]
scalar_params = [p for p in model.parameters() if p.ndim < 2]
head_params = [model.lm_head.weight]

# init the optimizer(s)
adam_params = [dict(params=head_params, lr=0.008), dict(params=embed_params, lr=0.6), dict(params=scalar_params, lr=0.04)]
# small adam epsilon by @YouJiacheng. this is an alternate method of fixing the world_size dependence
# discovered by @fernbear.bsky.social https://x.com/hi_tysam/status/1879692937589875094
optimizer1 = torch.optim.Adam(adam_params, betas=(0.8, 0.95), eps=1e-10, fused=True)
optimizer2 = Muon(hidden_matrix_params, lr=0.05, momentum=0.95, rank=rank, world_size=world_size)
optimizers = [optimizer1, optimizer2]

# learning rate schedule: stable then decay
def get_lr(step: int):
    x = step / args.num_iterations # progress in training
    assert 0 <= x <= 1
    w = min((1 - x) / args.cooldown_frac, 1.0) # 1 -> 0
    return w * 1.0 + (1 - w) * 0.1
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
@lru_cache(1)
def window_size_blocks(window_size: int):
    return torch.tensor(window_size // 128, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True)

model: nn.Module = torch.compile(model, dynamic=False)

# Warmup the training kernels, then re-initialize the state so we aren't cheating
warmup_steps = 10
initial_state = dict(model=copy.deepcopy(model.state_dict()),
                     optimizers=[copy.deepcopy(opt.state_dict()) for opt in optimizers]) # save the initial state
train_loader = distributed_data_generator(args.train_files, world_size * args.seq_len, rank, world_size)
for _ in range(warmup_steps):
    inputs, targets = next(train_loader)
    model(inputs, targets, window_size_blocks(128)).backward()
    for param in model.parameters():
        dist.all_reduce(param.grad, op=dist.ReduceOp.AVG)
    for opt in optimizers:
        opt.step()
    model.zero_grad(set_to_none=True)
model.load_state_dict(initial_state['model'])
for opt, opt_state in zip(optimizers, initial_state['optimizers']):
    opt.load_state_dict(opt_state)
del train_loader, initial_state

train_loader = distributed_data_generator(args.train_files, world_size * args.seq_len, rank, world_size)
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
train_steps = args.num_iterations
for step in range(train_steps + 1):
    last_step = (step == train_steps)

    # Linearly increase the block-wise sliding window size over training 128 -> 1792:
    # increase by @fernbear.bsky.social; block-wise by @YouJiacheng
    window_size = next_multiple_of_n(1728 * step / train_steps, n=128)

    # --------------- VALIDATION SECTION -----------------
    if last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        model.eval()
        val_batch_size = world_size * args.val_seq_len
        assert args.val_tokens % val_batch_size == 0
        val_steps = args.val_tokens // val_batch_size
        val_loader = distributed_data_generator(args.val_files, val_batch_size, rank, world_size)
        val_loss = 0
        with torch.no_grad():
            for _ in range(val_steps):
                inputs, targets = next(val_loader)
                val_loss += model(inputs, targets, window_size_blocks(window_size))
        val_loss /= val_steps
        del val_loader
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        print0(f"step:{step}/{train_steps} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/max(step, 1):.2f}ms", console=True)
        model.train()
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if last_step:
        if master_process and args.save_checkpoint:
            log = dict(step=step, code=code, model=model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
            os.makedirs(f"logs/{run_id}", exist_ok=True)
            torch.save(log, f"logs/{run_id}/state_step{step:06d}.pt")
        # the last step only has the validation loop, so break to avoid training
        break

    # --------------- TRAINING SECTION -----------------
    inputs, targets = next(train_loader)
    model(inputs, targets, window_size_blocks(window_size)).backward()
    for param in model.parameters():
        dist.all_reduce(param.grad, op=dist.ReduceOp.AVG)
    # momentum warmup for Muon
    frac = min(step / 300, 1)
    for group in optimizer2.param_groups:
        group["momentum"] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # logging
    approx_training_time_ms = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{train_steps} train_time:{approx_training_time_ms:.0f}ms step_avg:{approx_training_time_ms/(step + 1):.2f}ms", console=True)

print0(
    f"peak memory allocated: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB "
    f"reserved: {torch.cuda.max_memory_reserved() // 1024 // 1024} MiB",
    console=True,
)
dist.destroy_process_group()

====================================================================================================
Running Python 3.12.7 (main, Feb  1 2025, 03:09:49) [GCC 13.2.0]
Running PyTorch 2.7.0.dev20250125+cu126 compiled for CUDA 12.6
Sun Feb  2 03:23:40 2025       
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.127.08             Driver Version: 550.127.08     CUDA Version: 12.6     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA H100 80GB HBM3          On  |   00000000:61:00.0 Off |                    0 |
| N/A   27C    P0            122W /  700W |    7746MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  |   00000000:62:00.0 Off |                    0 |
| N/A   30C    P0            117W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  |   00000000:63:00.0 Off |                    0 |
| N/A   29C    P0            117W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  |   00000000:64:00.0 Off |                    0 |
| N/A   26C    P0            116W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  |   00000000:6A:00.0 Off |                    0 |
| N/A   26C    P0            124W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  |   00000000:6B:00.0 Off |                    0 |
| N/A   29C    P0            118W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  |   00000000:6C:00.0 Off |                    0 |
| N/A   28C    P0            119W /  700W |    3456MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  |   00000000:6D:00.0 Off |                    0 |
| N/A   26C    P0            118W /  700W |    3216MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
                                                                                         
+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
+-----------------------------------------------------------------------------------------+

====================================================================================================
step:0/1770 val_loss:10.8258 train_time:0ms step_avg:0.03ms
step:1/1770 train_time:70ms step_avg:70.50ms
step:2/1770 train_time:148ms step_avg:74.01ms
step:3/1770 train_time:237ms step_avg:78.90ms
step:4/1770 train_time:332ms step_avg:82.92ms
step:5/1770 train_time:428ms step_avg:85.56ms
step:6/1770 train_time:523ms step_avg:87.22ms
step:7/1770 train_time:619ms step_avg:88.47ms
step:8/1770 train_time:715ms step_avg:89.42ms
step:9/1770 train_time:811ms step_avg:90.12ms
step:10/1770 train_time:907ms step_avg:90.71ms
step:11/1770 train_time:1003ms step_avg:91.14ms
step:12/1770 train_time:1099ms step_avg:91.55ms
step:13/1770 train_time:1194ms step_avg:91.88ms
step:14/1770 train_time:1291ms step_avg:92.20ms
step:15/1770 train_time:1387ms step_avg:92.50ms
step:16/1770 train_time:1484ms step_avg:92.72ms
step:17/1770 train_time:1579ms step_avg:92.89ms
step:18/1770 train_time:1675ms step_avg:93.05ms
step:19/1770 train_time:1771ms step_avg:93.22ms
step:20/1770 train_time:1868ms step_avg:93.39ms
step:21/1770 train_time:1964ms step_avg:93.55ms
step:22/1770 train_time:2060ms step_avg:93.66ms
step:23/1770 train_time:2157ms step_avg:93.77ms
step:24/1770 train_time:2253ms step_avg:93.86ms
step:25/1770 train_time:2350ms step_avg:93.98ms
step:26/1770 train_time:2445ms step_avg:94.05ms
step:27/1770 train_time:2541ms step_avg:94.11ms
step:28/1770 train_time:2637ms step_avg:94.18ms
step:29/1770 train_time:2733ms step_avg:94.24ms
step:30/1770 train_time:2829ms step_avg:94.31ms
step:31/1770 train_time:2925ms step_avg:94.34ms
step:32/1770 train_time:3020ms step_avg:94.38ms
step:33/1770 train_time:3116ms step_avg:94.44ms
step:34/1770 train_time:3212ms step_avg:94.47ms
step:35/1770 train_time:3308ms step_avg:94.52ms
step:36/1770 train_time:3404ms step_avg:94.57ms
step:37/1770 train_time:3501ms step_avg:94.61ms
step:38/1770 train_time:3596ms step_avg:94.64ms
step:39/1770 train_time:3692ms step_avg:94.67ms
step:40/1770 train_time:3789ms step_avg:94.72ms
step:41/1770 train_time:3885ms step_avg:94.76ms
step:42/1770 train_time:3981ms step_avg:94.79ms
step:43/1770 train_time:4078ms step_avg:94.83ms
step:44/1770 train_time:4174ms step_avg:94.86ms
step:45/1770 train_time:4270ms step_avg:94.89ms
step:46/1770 train_time:4366ms step_avg:94.92ms
step:47/1770 train_time:4462ms step_avg:94.94ms
step:48/1770 train_time:4558ms step_avg:94.95ms
step:49/1770 train_time:4654ms step_avg:94.97ms
step:50/1770 train_time:4749ms step_avg:94.99ms
step:51/1770 train_time:4847ms step_avg:95.03ms
step:52/1770 train_time:4942ms step_avg:95.03ms
step:53/1770 train_time:5038ms step_avg:95.05ms
step:54/1770 train_time:5133ms step_avg:95.06ms
step:55/1770 train_time:5229ms step_avg:95.08ms
step:56/1770 train_time:5326ms step_avg:95.10ms
step:57/1770 train_time:5422ms step_avg:95.13ms
step:58/1770 train_time:5519ms step_avg:95.15ms
step:59/1770 train_time:5615ms step_avg:95.17ms
step:60/1770 train_time:5711ms step_avg:95.18ms
step:61/1770 train_time:5808ms step_avg:95.21ms
step:62/1770 train_time:5904ms step_avg:95.22ms
step:63/1770 train_time:6000ms step_avg:95.24ms
step:64/1770 train_time:6096ms step_avg:95.25ms
step:65/1770 train_time:6192ms step_avg:95.26ms
step:66/1770 train_time:6288ms step_avg:95.27ms
step:67/1770 train_time:6384ms step_avg:95.29ms
step:68/1770 train_time:6481ms step_avg:95.31ms
step:69/1770 train_time:6577ms step_avg:95.32ms
step:70/1770 train_time:6674ms step_avg:95.34ms
step:71/1770 train_time:6769ms step_avg:95.34ms
step:72/1770 train_time:6865ms step_avg:95.34ms
step:73/1770 train_time:6960ms step_avg:95.35ms
step:74/1770 train_time:7056ms step_avg:95.36ms
step:75/1770 train_time:7152ms step_avg:95.36ms
step:76/1770 train_time:7248ms step_avg:95.37ms
step:77/1770 train_time:7343ms step_avg:95.37ms
step:78/1770 train_time:7439ms step_avg:95.38ms
step:79/1770 train_time:7535ms step_avg:95.38ms
step:80/1770 train_time:7631ms step_avg:95.39ms
step:81/1770 train_time:7728ms step_avg:95.40ms
step:82/1770 train_time:7824ms step_avg:95.42ms
step:83/1770 train_time:7921ms step_avg:95.43ms
step:84/1770 train_time:8017ms step_avg:95.44ms
step:85/1770 train_time:8113ms step_avg:95.45ms
step:86/1770 train_time:8209ms step_avg:95.46ms
step:87/1770 train_time:8306ms step_avg:95.47ms
step:88/1770 train_time:8402ms step_avg:95.47ms
step:89/1770 train_time:8497ms step_avg:95.47ms
step:90/1770 train_time:8593ms step_avg:95.48ms
step:91/1770 train_time:8689ms step_avg:95.49ms
step:92/1770 train_time:8786ms step_avg:95.50ms
step:93/1770 train_time:8881ms step_avg:95.50ms
step:94/1770 train_time:8977ms step_avg:95.50ms
step:95/1770 train_time:9072ms step_avg:95.50ms
step:96/1770 train_time:9168ms step_avg:95.50ms
step:97/1770 train_time:9264ms step_avg:95.51ms
step:98/1770 train_time:9360ms step_avg:95.51ms
step:99/1770 train_time:9456ms step_avg:95.51ms
step:100/1770 train_time:9551ms step_avg:95.51ms
step:101/1770 train_time:9648ms step_avg:95.52ms
step:102/1770 train_time:9744ms step_avg:95.53ms
step:103/1770 train_time:9840ms step_avg:95.53ms
step:104/1770 train_time:9936ms step_avg:95.53ms
step:105/1770 train_time:10031ms step_avg:95.54ms
step:106/1770 train_time:10127ms step_avg:95.54ms
step:107/1770 train_time:10224ms step_avg:95.55ms
step:108/1770 train_time:10320ms step_avg:95.56ms
step:109/1770 train_time:10418ms step_avg:95.58ms
step:110/1770 train_time:10514ms step_avg:95.58ms
step:111/1770 train_time:10610ms step_avg:95.58ms
step:112/1770 train_time:10707ms step_avg:95.59ms
step:113/1770 train_time:10802ms step_avg:95.59ms
step:114/1770 train_time:10898ms step_avg:95.59ms
step:115/1770 train_time:10993ms step_avg:95.59ms
step:116/1770 train_time:11089ms step_avg:95.60ms
step:117/1770 train_time:11185ms step_avg:95.60ms
step:118/1770 train_time:11282ms step_avg:95.61ms
step:119/1770 train_time:11378ms step_avg:95.61ms
step:120/1770 train_time:11474ms step_avg:95.62ms
step:121/1770 train_time:11570ms step_avg:95.62ms
step:122/1770 train_time:11666ms step_avg:95.62ms
step:123/1770 train_time:11762ms step_avg:95.63ms
step:124/1770 train_time:11858ms step_avg:95.63ms
step:125/1770 train_time:11954ms step_avg:95.63ms
step:125/1770 val_loss:4.6539 train_time:12048ms step_avg:96.38ms
step:126/1770 train_time:12070ms step_avg:95.79ms
step:127/1770 train_time:12152ms step_avg:95.69ms
step:128/1770 train_time:12254ms step_avg:95.73ms
step:129/1770 train_time:12350ms step_avg:95.74ms
step:130/1770 train_time:12446ms step_avg:95.74ms
step:131/1770 train_time:12542ms step_avg:95.74ms
step:132/1770 train_time:12638ms step_avg:95.74ms
step:133/1770 train_time:12733ms step_avg:95.74ms
step:134/1770 train_time:12830ms step_avg:95.75ms
step:135/1770 train_time:12926ms step_avg:95.75ms
step:136/1770 train_time:13022ms step_avg:95.75ms
step:137/1770 train_time:13119ms step_avg:95.76ms
step:138/1770 train_time:13216ms step_avg:95.76ms
step:139/1770 train_time:13312ms step_avg:95.77ms
step:140/1770 train_time:13409ms step_avg:95.78ms
step:141/1770 train_time:13506ms step_avg:95.79ms
step:142/1770 train_time:13603ms step_avg:95.80ms
step:143/1770 train_time:13700ms step_avg:95.80ms
step:144/1770 train_time:13796ms step_avg:95.81ms
step:145/1770 train_time:13893ms step_avg:95.81ms
step:146/1770 train_time:13989ms step_avg:95.82ms
step:147/1770 train_time:14086ms step_avg:95.83ms
step:148/1770 train_time:14184ms step_avg:95.84ms
step:149/1770 train_time:14280ms step_avg:95.84ms
step:150/1770 train_time:14376ms step_avg:95.84ms
step:151/1770 train_time:14473ms step_avg:95.85ms
step:152/1770 train_time:14570ms step_avg:95.86ms
step:153/1770 train_time:14667ms step_avg:95.87ms
step:154/1770 train_time:14765ms step_avg:95.88ms
step:155/1770 train_time:14861ms step_avg:95.88ms
step:156/1770 train_time:14957ms step_avg:95.88ms
step:157/1770 train_time:15054ms step_avg:95.88ms
step:158/1770 train_time:15151ms step_avg:95.89ms
step:159/1770 train_time:15248ms step_avg:95.90ms
step:160/1770 train_time:15343ms step_avg:95.89ms
step:161/1770 train_time:15439ms step_avg:95.90ms
step:162/1770 train_time:15535ms step_avg:95.90ms
step:163/1770 train_time:15631ms step_avg:95.90ms
step:164/1770 train_time:15729ms step_avg:95.91ms
step:165/1770 train_time:15826ms step_avg:95.91ms
step:166/1770 train_time:15922ms step_avg:95.92ms
step:167/1770 train_time:16019ms step_avg:95.92ms
step:168/1770 train_time:16116ms step_avg:95.93ms
step:169/1770 train_time:16212ms step_avg:95.93ms
step:170/1770 train_time:16309ms step_avg:95.94ms
step:171/1770 train_time:16407ms step_avg:95.95ms
step:172/1770 train_time:16503ms step_avg:95.95ms
step:173/1770 train_time:16599ms step_avg:95.95ms
step:174/1770 train_time:16695ms step_avg:95.95ms
step:175/1770 train_time:16791ms step_avg:95.95ms
step:176/1770 train_time:16888ms step_avg:95.96ms
step:177/1770 train_time:16985ms step_avg:95.96ms
step:178/1770 train_time:17082ms step_avg:95.97ms
step:179/1770 train_time:17178ms step_avg:95.97ms
step:180/1770 train_time:17274ms step_avg:95.97ms
step:181/1770 train_time:17371ms step_avg:95.97ms
step:182/1770 train_time:17467ms step_avg:95.97ms
step:183/1770 train_time:17564ms step_avg:95.98ms
step:184/1770 train_time:17661ms step_avg:95.98ms
step:185/1770 train_time:17757ms step_avg:95.98ms
step:186/1770 train_time:17853ms step_avg:95.98ms
step:187/1770 train_time:17950ms step_avg:95.99ms
step:188/1770 train_time:18047ms step_avg:96.00ms
step:189/1770 train_time:18145ms step_avg:96.00ms
step:190/1770 train_time:18241ms step_avg:96.01ms
step:191/1770 train_time:18338ms step_avg:96.01ms
step:192/1770 train_time:18435ms step_avg:96.01ms
step:193/1770 train_time:18531ms step_avg:96.02ms
step:194/1770 train_time:18628ms step_avg:96.02ms
step:195/1770 train_time:18725ms step_avg:96.02ms
step:196/1770 train_time:18821ms step_avg:96.03ms
step:197/1770 train_time:18918ms step_avg:96.03ms
step:198/1770 train_time:19016ms step_avg:96.04ms
step:199/1770 train_time:19112ms step_avg:96.04ms
step:200/1770 train_time:19209ms step_avg:96.05ms
step:201/1770 train_time:19306ms step_avg:96.05ms
step:202/1770 train_time:19402ms step_avg:96.05ms
step:203/1770 train_time:19498ms step_avg:96.05ms
step:204/1770 train_time:19594ms step_avg:96.05ms
step:205/1770 train_time:19691ms step_avg:96.05ms
step:206/1770 train_time:19788ms step_avg:96.06ms
step:207/1770 train_time:19885ms step_avg:96.06ms
step:208/1770 train_time:19981ms step_avg:96.06ms
step:209/1770 train_time:20078ms step_avg:96.07ms
step:210/1770 train_time:20175ms step_avg:96.07ms
step:211/1770 train_time:20271ms step_avg:96.07ms
step:212/1770 train_time:20367ms step_avg:96.07ms
step:213/1770 train_time:20464ms step_avg:96.07ms
step:214/1770 train_time:20560ms step_avg:96.08ms
step:215/1770 train_time:20657ms step_avg:96.08ms
step:216/1770 train_time:20753ms step_avg:96.08ms
step:217/1770 train_time:20850ms step_avg:96.08ms
step:218/1770 train_time:20946ms step_avg:96.08ms
step:219/1770 train_time:21042ms step_avg:96.08ms
step:220/1770 train_time:21138ms step_avg:96.08ms
step:221/1770 train_time:21235ms step_avg:96.08ms
step:222/1770 train_time:21331ms step_avg:96.09ms
step:223/1770 train_time:21428ms step_avg:96.09ms
step:224/1770 train_time:21525ms step_avg:96.09ms
step:225/1770 train_time:21622ms step_avg:96.10ms
step:226/1770 train_time:21718ms step_avg:96.10ms
step:227/1770 train_time:21815ms step_avg:96.10ms
step:228/1770 train_time:21912ms step_avg:96.11ms
step:229/1770 train_time:22009ms step_avg:96.11ms
step:230/1770 train_time:22106ms step_avg:96.11ms
step:231/1770 train_time:22202ms step_avg:96.11ms
step:232/1770 train_time:22299ms step_avg:96.12ms
step:233/1770 train_time:22395ms step_avg:96.11ms
step:234/1770 train_time:22491ms step_avg:96.12ms
step:235/1770 train_time:22588ms step_avg:96.12ms
step:236/1770 train_time:22685ms step_avg:96.12ms
step:237/1770 train_time:22781ms step_avg:96.12ms
step:238/1770 train_time:22878ms step_avg:96.12ms
step:239/1770 train_time:22974ms step_avg:96.13ms
step:240/1770 train_time:23071ms step_avg:96.13ms
step:241/1770 train_time:23168ms step_avg:96.13ms
step:242/1770 train_time:23265ms step_avg:96.13ms
step:243/1770 train_time:23361ms step_avg:96.14ms
step:244/1770 train_time:23458ms step_avg:96.14ms
step:245/1770 train_time:23555ms step_avg:96.14ms
step:246/1770 train_time:23651ms step_avg:96.14ms
step:247/1770 train_time:23749ms step_avg:96.15ms
step:248/1770 train_time:23844ms step_avg:96.15ms
step:249/1770 train_time:23941ms step_avg:96.15ms
step:250/1770 train_time:24037ms step_avg:96.15ms
step:250/1770 val_loss:4.1116 train_time:24131ms step_avg:96.53ms
step:251/1770 train_time:24153ms step_avg:96.23ms
step:252/1770 train_time:24237ms step_avg:96.18ms
step:253/1770 train_time:24333ms step_avg:96.18ms
step:254/1770 train_time:24430ms step_avg:96.18ms
step:255/1770 train_time:24527ms step_avg:96.18ms
step:256/1770 train_time:24623ms step_avg:96.18ms
step:257/1770 train_time:24719ms step_avg:96.18ms
step:258/1770 train_time:24815ms step_avg:96.18ms
step:259/1770 train_time:24912ms step_avg:96.18ms
step:260/1770 train_time:25008ms step_avg:96.19ms
step:261/1770 train_time:25104ms step_avg:96.18ms
step:262/1770 train_time:25201ms step_avg:96.19ms
step:263/1770 train_time:25297ms step_avg:96.19ms
step:264/1770 train_time:25395ms step_avg:96.19ms
step:265/1770 train_time:25492ms step_avg:96.19ms
step:266/1770 train_time:25589ms step_avg:96.20ms
step:267/1770 train_time:25687ms step_avg:96.21ms
step:268/1770 train_time:25785ms step_avg:96.21ms
step:269/1770 train_time:25882ms step_avg:96.21ms
step:270/1770 train_time:25979ms step_avg:96.22ms
step:271/1770 train_time:26077ms step_avg:96.22ms
step:272/1770 train_time:26173ms step_avg:96.22ms
step:273/1770 train_time:26270ms step_avg:96.23ms
step:274/1770 train_time:26369ms step_avg:96.24ms
step:275/1770 train_time:26466ms step_avg:96.24ms
step:276/1770 train_time:26564ms step_avg:96.24ms
step:277/1770 train_time:26661ms step_avg:96.25ms
step:278/1770 train_time:26758ms step_avg:96.25ms
step:279/1770 train_time:26855ms step_avg:96.25ms
step:280/1770 train_time:26951ms step_avg:96.26ms
step:281/1770 train_time:27049ms step_avg:96.26ms
step:282/1770 train_time:27147ms step_avg:96.27ms
step:283/1770 train_time:27244ms step_avg:96.27ms
step:284/1770 train_time:27340ms step_avg:96.27ms
step:285/1770 train_time:27436ms step_avg:96.27ms
step:286/1770 train_time:27533ms step_avg:96.27ms
step:287/1770 train_time:27630ms step_avg:96.27ms
step:288/1770 train_time:27728ms step_avg:96.28ms
step:289/1770 train_time:27826ms step_avg:96.28ms
step:290/1770 train_time:27923ms step_avg:96.29ms
step:291/1770 train_time:28021ms step_avg:96.29ms
step:292/1770 train_time:28118ms step_avg:96.29ms
step:293/1770 train_time:28215ms step_avg:96.30ms
step:294/1770 train_time:28313ms step_avg:96.30ms
step:295/1770 train_time:28410ms step_avg:96.30ms
step:296/1770 train_time:28507ms step_avg:96.31ms
step:297/1770 train_time:28605ms step_avg:96.31ms
step:298/1770 train_time:28701ms step_avg:96.31ms
step:299/1770 train_time:28799ms step_avg:96.32ms
step:300/1770 train_time:28895ms step_avg:96.32ms
step:301/1770 train_time:28993ms step_avg:96.32ms
step:302/1770 train_time:29091ms step_avg:96.33ms
step:303/1770 train_time:29189ms step_avg:96.33ms
step:304/1770 train_time:29285ms step_avg:96.33ms
step:305/1770 train_time:29383ms step_avg:96.34ms
step:306/1770 train_time:29480ms step_avg:96.34ms
step:307/1770 train_time:29577ms step_avg:96.34ms
step:308/1770 train_time:29674ms step_avg:96.34ms
step:309/1770 train_time:29771ms step_avg:96.35ms
step:310/1770 train_time:29869ms step_avg:96.35ms
step:311/1770 train_time:29967ms step_avg:96.36ms
step:312/1770 train_time:30064ms step_avg:96.36ms
step:313/1770 train_time:30162ms step_avg:96.37ms
step:314/1770 train_time:30259ms step_avg:96.37ms
step:315/1770 train_time:30356ms step_avg:96.37ms
step:316/1770 train_time:30453ms step_avg:96.37ms
step:317/1770 train_time:30550ms step_avg:96.37ms
step:318/1770 train_time:30648ms step_avg:96.38ms
step:319/1770 train_time:30746ms step_avg:96.38ms
step:320/1770 train_time:30843ms step_avg:96.39ms
step:321/1770 train_time:30940ms step_avg:96.38ms
step:322/1770 train_time:31037ms step_avg:96.39ms
step:323/1770 train_time:31135ms step_avg:96.39ms
step:324/1770 train_time:31232ms step_avg:96.40ms
step:325/1770 train_time:31330ms step_avg:96.40ms
step:326/1770 train_time:31427ms step_avg:96.40ms
step:327/1770 train_time:31525ms step_avg:96.41ms
step:328/1770 train_time:31622ms step_avg:96.41ms
step:329/1770 train_time:31719ms step_avg:96.41ms
step:330/1770 train_time:31815ms step_avg:96.41ms
step:331/1770 train_time:31912ms step_avg:96.41ms
step:332/1770 train_time:32010ms step_avg:96.42ms
step:333/1770 train_time:32108ms step_avg:96.42ms
step:334/1770 train_time:32206ms step_avg:96.43ms
step:335/1770 train_time:32303ms step_avg:96.43ms
step:336/1770 train_time:32400ms step_avg:96.43ms
step:337/1770 train_time:32497ms step_avg:96.43ms
step:338/1770 train_time:32593ms step_avg:96.43ms
step:339/1770 train_time:32691ms step_avg:96.43ms
step:340/1770 train_time:32789ms step_avg:96.44ms
step:341/1770 train_time:32887ms step_avg:96.44ms
step:342/1770 train_time:32985ms step_avg:96.45ms
step:343/1770 train_time:33082ms step_avg:96.45ms
step:344/1770 train_time:33179ms step_avg:96.45ms
step:345/1770 train_time:33276ms step_avg:96.45ms
step:346/1770 train_time:33373ms step_avg:96.45ms
step:347/1770 train_time:33470ms step_avg:96.46ms
step:348/1770 train_time:33567ms step_avg:96.46ms
step:349/1770 train_time:33664ms step_avg:96.46ms
step:350/1770 train_time:33762ms step_avg:96.46ms
step:351/1770 train_time:33859ms step_avg:96.46ms
step:352/1770 train_time:33955ms step_avg:96.46ms
step:353/1770 train_time:34052ms step_avg:96.47ms
step:354/1770 train_time:34149ms step_avg:96.47ms
step:355/1770 train_time:34247ms step_avg:96.47ms
step:356/1770 train_time:34344ms step_avg:96.47ms
step:357/1770 train_time:34441ms step_avg:96.47ms
step:358/1770 train_time:34537ms step_avg:96.47ms
step:359/1770 train_time:34634ms step_avg:96.47ms
step:360/1770 train_time:34732ms step_avg:96.48ms
step:361/1770 train_time:34830ms step_avg:96.48ms
step:362/1770 train_time:34929ms step_avg:96.49ms
step:363/1770 train_time:35026ms step_avg:96.49ms
step:364/1770 train_time:35123ms step_avg:96.49ms
step:365/1770 train_time:35220ms step_avg:96.49ms
step:366/1770 train_time:35316ms step_avg:96.49ms
step:367/1770 train_time:35413ms step_avg:96.49ms
step:368/1770 train_time:35510ms step_avg:96.49ms
step:369/1770 train_time:35608ms step_avg:96.50ms
step:370/1770 train_time:35705ms step_avg:96.50ms
step:371/1770 train_time:35803ms step_avg:96.50ms
step:372/1770 train_time:35900ms step_avg:96.51ms
step:373/1770 train_time:35997ms step_avg:96.51ms
step:374/1770 train_time:36094ms step_avg:96.51ms
step:375/1770 train_time:36192ms step_avg:96.51ms
step:375/1770 val_loss:3.9070 train_time:36287ms step_avg:96.77ms
step:376/1770 train_time:36309ms step_avg:96.57ms
step:377/1770 train_time:36393ms step_avg:96.53ms
step:378/1770 train_time:36493ms step_avg:96.54ms
step:379/1770 train_time:36591ms step_avg:96.55ms
step:380/1770 train_time:36688ms step_avg:96.55ms
step:381/1770 train_time:36785ms step_avg:96.55ms
step:382/1770 train_time:36881ms step_avg:96.55ms
step:383/1770 train_time:36978ms step_avg:96.55ms
step:384/1770 train_time:37074ms step_avg:96.55ms
step:385/1770 train_time:37171ms step_avg:96.55ms
step:386/1770 train_time:37268ms step_avg:96.55ms
step:387/1770 train_time:37365ms step_avg:96.55ms
step:388/1770 train_time:37464ms step_avg:96.56ms
step:389/1770 train_time:37562ms step_avg:96.56ms
step:390/1770 train_time:37659ms step_avg:96.56ms
step:391/1770 train_time:37756ms step_avg:96.56ms
step:392/1770 train_time:37854ms step_avg:96.57ms
step:393/1770 train_time:37950ms step_avg:96.56ms
step:394/1770 train_time:38047ms step_avg:96.57ms
step:395/1770 train_time:38144ms step_avg:96.57ms
step:396/1770 train_time:38244ms step_avg:96.57ms
step:397/1770 train_time:38342ms step_avg:96.58ms
step:398/1770 train_time:38441ms step_avg:96.58ms
step:399/1770 train_time:38540ms step_avg:96.59ms
step:400/1770 train_time:38639ms step_avg:96.60ms
step:401/1770 train_time:38738ms step_avg:96.60ms
step:402/1770 train_time:38838ms step_avg:96.61ms
step:403/1770 train_time:38937ms step_avg:96.62ms
step:404/1770 train_time:39036ms step_avg:96.62ms
step:405/1770 train_time:39136ms step_avg:96.63ms
step:406/1770 train_time:39236ms step_avg:96.64ms
step:407/1770 train_time:39337ms step_avg:96.65ms
step:408/1770 train_time:39437ms step_avg:96.66ms
step:409/1770 train_time:39536ms step_avg:96.67ms
step:410/1770 train_time:39636ms step_avg:96.67ms
step:411/1770 train_time:39734ms step_avg:96.68ms
step:412/1770 train_time:39833ms step_avg:96.68ms
step:413/1770 train_time:39931ms step_avg:96.69ms
step:414/1770 train_time:40030ms step_avg:96.69ms
step:415/1770 train_time:40130ms step_avg:96.70ms
step:416/1770 train_time:40229ms step_avg:96.70ms
step:417/1770 train_time:40329ms step_avg:96.71ms
step:418/1770 train_time:40428ms step_avg:96.72ms
step:419/1770 train_time:40528ms step_avg:96.73ms
step:420/1770 train_time:40628ms step_avg:96.73ms
step:421/1770 train_time:40728ms step_avg:96.74ms
step:422/1770 train_time:40829ms step_avg:96.75ms
step:423/1770 train_time:40929ms step_avg:96.76ms
step:424/1770 train_time:41028ms step_avg:96.77ms
step:425/1770 train_time:41128ms step_avg:96.77ms
step:426/1770 train_time:41227ms step_avg:96.78ms
step:427/1770 train_time:41326ms step_avg:96.78ms
step:428/1770 train_time:41425ms step_avg:96.79ms
step:429/1770 train_time:41525ms step_avg:96.80ms
step:430/1770 train_time:41625ms step_avg:96.80ms
step:431/1770 train_time:41726ms step_avg:96.81ms
step:432/1770 train_time:41827ms step_avg:96.82ms
step:433/1770 train_time:41928ms step_avg:96.83ms
step:434/1770 train_time:42029ms step_avg:96.84ms
step:435/1770 train_time:42129ms step_avg:96.85ms
step:436/1770 train_time:42228ms step_avg:96.85ms
step:437/1770 train_time:42327ms step_avg:96.86ms
step:438/1770 train_time:42427ms step_avg:96.86ms
step:439/1770 train_time:42526ms step_avg:96.87ms
step:440/1770 train_time:42626ms step_avg:96.88ms
step:441/1770 train_time:42726ms step_avg:96.88ms
step:442/1770 train_time:42826ms step_avg:96.89ms
step:443/1770 train_time:42928ms step_avg:96.90ms
step:444/1770 train_time:43028ms step_avg:96.91ms
step:445/1770 train_time:43128ms step_avg:96.92ms
step:446/1770 train_time:43226ms step_avg:96.92ms
step:447/1770 train_time:43326ms step_avg:96.93ms
step:448/1770 train_time:43425ms step_avg:96.93ms
step:449/1770 train_time:43524ms step_avg:96.93ms
step:450/1770 train_time:43623ms step_avg:96.94ms
step:451/1770 train_time:43722ms step_avg:96.94ms
step:452/1770 train_time:43821ms step_avg:96.95ms
step:453/1770 train_time:43920ms step_avg:96.95ms
step:454/1770 train_time:44020ms step_avg:96.96ms
step:455/1770 train_time:44120ms step_avg:96.97ms
step:456/1770 train_time:44219ms step_avg:96.97ms
step:457/1770 train_time:44318ms step_avg:96.97ms
step:458/1770 train_time:44417ms step_avg:96.98ms
step:459/1770 train_time:44518ms step_avg:96.99ms
step:460/1770 train_time:44618ms step_avg:97.00ms
step:461/1770 train_time:44717ms step_avg:97.00ms
step:462/1770 train_time:44816ms step_avg:97.00ms
step:463/1770 train_time:44915ms step_avg:97.01ms
step:464/1770 train_time:45015ms step_avg:97.02ms
step:465/1770 train_time:45114ms step_avg:97.02ms
step:466/1770 train_time:45213ms step_avg:97.02ms
step:467/1770 train_time:45312ms step_avg:97.03ms
step:468/1770 train_time:45412ms step_avg:97.03ms
step:469/1770 train_time:45512ms step_avg:97.04ms
step:470/1770 train_time:45612ms step_avg:97.05ms
step:471/1770 train_time:45711ms step_avg:97.05ms
step:472/1770 train_time:45811ms step_avg:97.06ms
step:473/1770 train_time:45910ms step_avg:97.06ms
step:474/1770 train_time:46010ms step_avg:97.07ms
step:475/1770 train_time:46109ms step_avg:97.07ms
step:476/1770 train_time:46209ms step_avg:97.08ms
step:477/1770 train_time:46308ms step_avg:97.08ms
step:478/1770 train_time:46408ms step_avg:97.09ms
step:479/1770 train_time:46508ms step_avg:97.09ms
step:480/1770 train_time:46608ms step_avg:97.10ms
step:481/1770 train_time:46708ms step_avg:97.11ms
step:482/1770 train_time:46808ms step_avg:97.11ms
step:483/1770 train_time:46908ms step_avg:97.12ms
step:484/1770 train_time:47008ms step_avg:97.12ms
step:485/1770 train_time:47108ms step_avg:97.13ms
step:486/1770 train_time:47208ms step_avg:97.13ms
step:487/1770 train_time:47307ms step_avg:97.14ms
step:488/1770 train_time:47407ms step_avg:97.15ms
step:489/1770 train_time:47508ms step_avg:97.15ms
step:490/1770 train_time:47607ms step_avg:97.16ms
step:491/1770 train_time:47707ms step_avg:97.16ms
step:492/1770 train_time:47807ms step_avg:97.17ms
step:493/1770 train_time:47908ms step_avg:97.18ms
step:494/1770 train_time:48007ms step_avg:97.18ms
step:495/1770 train_time:48107ms step_avg:97.19ms
step:496/1770 train_time:48206ms step_avg:97.19ms
step:497/1770 train_time:48306ms step_avg:97.20ms
step:498/1770 train_time:48406ms step_avg:97.20ms
step:499/1770 train_time:48506ms step_avg:97.21ms
step:500/1770 train_time:48606ms step_avg:97.21ms
step:500/1770 val_loss:3.7528 train_time:48704ms step_avg:97.41ms
step:501/1770 train_time:48725ms step_avg:97.26ms
step:502/1770 train_time:48814ms step_avg:97.24ms
step:503/1770 train_time:48913ms step_avg:97.24ms
step:504/1770 train_time:49013ms step_avg:97.25ms
step:505/1770 train_time:49111ms step_avg:97.25ms
step:506/1770 train_time:49210ms step_avg:97.25ms
step:507/1770 train_time:49309ms step_avg:97.26ms
step:508/1770 train_time:49408ms step_avg:97.26ms
step:509/1770 train_time:49507ms step_avg:97.26ms
step:510/1770 train_time:49606ms step_avg:97.27ms
step:511/1770 train_time:49705ms step_avg:97.27ms
step:512/1770 train_time:49805ms step_avg:97.28ms
step:513/1770 train_time:49906ms step_avg:97.28ms
step:514/1770 train_time:50006ms step_avg:97.29ms
step:515/1770 train_time:50107ms step_avg:97.30ms
step:516/1770 train_time:50207ms step_avg:97.30ms
step:517/1770 train_time:50306ms step_avg:97.30ms
step:518/1770 train_time:50406ms step_avg:97.31ms
step:519/1770 train_time:50504ms step_avg:97.31ms
step:520/1770 train_time:50603ms step_avg:97.31ms
step:521/1770 train_time:50702ms step_avg:97.32ms
step:522/1770 train_time:50801ms step_avg:97.32ms
step:523/1770 train_time:50901ms step_avg:97.32ms
step:524/1770 train_time:51001ms step_avg:97.33ms
step:525/1770 train_time:51100ms step_avg:97.33ms
step:526/1770 train_time:51200ms step_avg:97.34ms
step:527/1770 train_time:51299ms step_avg:97.34ms
step:528/1770 train_time:51399ms step_avg:97.35ms
step:529/1770 train_time:51498ms step_avg:97.35ms
step:530/1770 train_time:51597ms step_avg:97.35ms
step:531/1770 train_time:51697ms step_avg:97.36ms
step:532/1770 train_time:51797ms step_avg:97.36ms
step:533/1770 train_time:51897ms step_avg:97.37ms
step:534/1770 train_time:51997ms step_avg:97.37ms
step:535/1770 train_time:52097ms step_avg:97.38ms
step:536/1770 train_time:52197ms step_avg:97.38ms
step:537/1770 train_time:52298ms step_avg:97.39ms
step:538/1770 train_time:52398ms step_avg:97.39ms
step:539/1770 train_time:52497ms step_avg:97.40ms
step:540/1770 train_time:52597ms step_avg:97.40ms
step:541/1770 train_time:52696ms step_avg:97.40ms
step:542/1770 train_time:52795ms step_avg:97.41ms
step:543/1770 train_time:52894ms step_avg:97.41ms
step:544/1770 train_time:52993ms step_avg:97.41ms
step:545/1770 train_time:53092ms step_avg:97.42ms
step:546/1770 train_time:53192ms step_avg:97.42ms
step:547/1770 train_time:53291ms step_avg:97.42ms
step:548/1770 train_time:53390ms step_avg:97.43ms
step:549/1770 train_time:53490ms step_avg:97.43ms
step:550/1770 train_time:53590ms step_avg:97.44ms
step:551/1770 train_time:53690ms step_avg:97.44ms
step:552/1770 train_time:53791ms step_avg:97.45ms
step:553/1770 train_time:53890ms step_avg:97.45ms
step:554/1770 train_time:53990ms step_avg:97.46ms
step:555/1770 train_time:54090ms step_avg:97.46ms
step:556/1770 train_time:54190ms step_avg:97.46ms
step:557/1770 train_time:54290ms step_avg:97.47ms
step:558/1770 train_time:54390ms step_avg:97.47ms
step:559/1770 train_time:54490ms step_avg:97.48ms
step:560/1770 train_time:54590ms step_avg:97.48ms
step:561/1770 train_time:54690ms step_avg:97.49ms
step:562/1770 train_time:54790ms step_avg:97.49ms
step:563/1770 train_time:54890ms step_avg:97.50ms
step:564/1770 train_time:54990ms step_avg:97.50ms
step:565/1770 train_time:55090ms step_avg:97.50ms
step:566/1770 train_time:55190ms step_avg:97.51ms
step:567/1770 train_time:55289ms step_avg:97.51ms
step:568/1770 train_time:55390ms step_avg:97.52ms
step:569/1770 train_time:55489ms step_avg:97.52ms
step:570/1770 train_time:55589ms step_avg:97.53ms
step:571/1770 train_time:55689ms step_avg:97.53ms
step:572/1770 train_time:55790ms step_avg:97.53ms
step:573/1770 train_time:55890ms step_avg:97.54ms
step:574/1770 train_time:55990ms step_avg:97.54ms
step:575/1770 train_time:56090ms step_avg:97.55ms
step:576/1770 train_time:56190ms step_avg:97.55ms
step:577/1770 train_time:56290ms step_avg:97.56ms
step:578/1770 train_time:56389ms step_avg:97.56ms
step:579/1770 train_time:56489ms step_avg:97.56ms
step:580/1770 train_time:56589ms step_avg:97.57ms
step:581/1770 train_time:56689ms step_avg:97.57ms
step:582/1770 train_time:56789ms step_avg:97.58ms
step:583/1770 train_time:56889ms step_avg:97.58ms
step:584/1770 train_time:56989ms step_avg:97.58ms
step:585/1770 train_time:57089ms step_avg:97.59ms
step:586/1770 train_time:57189ms step_avg:97.59ms
step:587/1770 train_time:57289ms step_avg:97.60ms
step:588/1770 train_time:57388ms step_avg:97.60ms
step:589/1770 train_time:57488ms step_avg:97.60ms
step:590/1770 train_time:57588ms step_avg:97.61ms
step:591/1770 train_time:57688ms step_avg:97.61ms
step:592/1770 train_time:57789ms step_avg:97.62ms
step:593/1770 train_time:57889ms step_avg:97.62ms
step:594/1770 train_time:57989ms step_avg:97.62ms
step:595/1770 train_time:58089ms step_avg:97.63ms
step:596/1770 train_time:58189ms step_avg:97.63ms
step:597/1770 train_time:58289ms step_avg:97.64ms
step:598/1770 train_time:58389ms step_avg:97.64ms
step:599/1770 train_time:58489ms step_avg:97.64ms
step:600/1770 train_time:58589ms step_avg:97.65ms
step:601/1770 train_time:58689ms step_avg:97.65ms
step:602/1770 train_time:58790ms step_avg:97.66ms
step:603/1770 train_time:58889ms step_avg:97.66ms
step:604/1770 train_time:58989ms step_avg:97.66ms
step:605/1770 train_time:59089ms step_avg:97.67ms
step:606/1770 train_time:59189ms step_avg:97.67ms
step:607/1770 train_time:59290ms step_avg:97.68ms
step:608/1770 train_time:59389ms step_avg:97.68ms
step:609/1770 train_time:59489ms step_avg:97.68ms
step:610/1770 train_time:59589ms step_avg:97.69ms
step:611/1770 train_time:59689ms step_avg:97.69ms
step:612/1770 train_time:59789ms step_avg:97.69ms
step:613/1770 train_time:59889ms step_avg:97.70ms
step:614/1770 train_time:59989ms step_avg:97.70ms
step:615/1770 train_time:60089ms step_avg:97.71ms
step:616/1770 train_time:60189ms step_avg:97.71ms
step:617/1770 train_time:60289ms step_avg:97.71ms
step:618/1770 train_time:60389ms step_avg:97.72ms
step:619/1770 train_time:60490ms step_avg:97.72ms
step:620/1770 train_time:60590ms step_avg:97.73ms
step:621/1770 train_time:60690ms step_avg:97.73ms
step:622/1770 train_time:60790ms step_avg:97.73ms
step:623/1770 train_time:60890ms step_avg:97.74ms
step:624/1770 train_time:60990ms step_avg:97.74ms
step:625/1770 train_time:61090ms step_avg:97.74ms
step:625/1770 val_loss:3.6659 train_time:61188ms step_avg:97.90ms
step:626/1770 train_time:61209ms step_avg:97.78ms
step:627/1770 train_time:61297ms step_avg:97.76ms
step:628/1770 train_time:61400ms step_avg:97.77ms
step:629/1770 train_time:61500ms step_avg:97.77ms
step:630/1770 train_time:61599ms step_avg:97.78ms
step:631/1770 train_time:61698ms step_avg:97.78ms
step:632/1770 train_time:61796ms step_avg:97.78ms
step:633/1770 train_time:61896ms step_avg:97.78ms
step:634/1770 train_time:61995ms step_avg:97.78ms
step:635/1770 train_time:62093ms step_avg:97.78ms
step:636/1770 train_time:62193ms step_avg:97.79ms
step:637/1770 train_time:62293ms step_avg:97.79ms
step:638/1770 train_time:62394ms step_avg:97.80ms
step:639/1770 train_time:62495ms step_avg:97.80ms
step:640/1770 train_time:62594ms step_avg:97.80ms
step:641/1770 train_time:62694ms step_avg:97.81ms
step:642/1770 train_time:62793ms step_avg:97.81ms
step:643/1770 train_time:62893ms step_avg:97.81ms
step:644/1770 train_time:62993ms step_avg:97.81ms
step:645/1770 train_time:63092ms step_avg:97.82ms
step:646/1770 train_time:63191ms step_avg:97.82ms
step:647/1770 train_time:63291ms step_avg:97.82ms
step:648/1770 train_time:63390ms step_avg:97.82ms
step:649/1770 train_time:63490ms step_avg:97.83ms
step:650/1770 train_time:63590ms step_avg:97.83ms
step:651/1770 train_time:63689ms step_avg:97.83ms
step:652/1770 train_time:63788ms step_avg:97.83ms
step:653/1770 train_time:63888ms step_avg:97.84ms
step:654/1770 train_time:63988ms step_avg:97.84ms
step:655/1770 train_time:64088ms step_avg:97.84ms
step:656/1770 train_time:64188ms step_avg:97.85ms
step:657/1770 train_time:64288ms step_avg:97.85ms
step:658/1770 train_time:64390ms step_avg:97.86ms
step:659/1770 train_time:64491ms step_avg:97.86ms
step:660/1770 train_time:64592ms step_avg:97.87ms
step:661/1770 train_time:64693ms step_avg:97.87ms
step:662/1770 train_time:64795ms step_avg:97.88ms
step:663/1770 train_time:64895ms step_avg:97.88ms
step:664/1770 train_time:64997ms step_avg:97.89ms
step:665/1770 train_time:65098ms step_avg:97.89ms
step:666/1770 train_time:65200ms step_avg:97.90ms
step:667/1770 train_time:65300ms step_avg:97.90ms
step:668/1770 train_time:65401ms step_avg:97.91ms
step:669/1770 train_time:65502ms step_avg:97.91ms
step:670/1770 train_time:65603ms step_avg:97.92ms
step:671/1770 train_time:65705ms step_avg:97.92ms
step:672/1770 train_time:65806ms step_avg:97.93ms
step:673/1770 train_time:65909ms step_avg:97.93ms
step:674/1770 train_time:66010ms step_avg:97.94ms
step:675/1770 train_time:66112ms step_avg:97.94ms
step:676/1770 train_time:66213ms step_avg:97.95ms
step:677/1770 train_time:66315ms step_avg:97.95ms
step:678/1770 train_time:66416ms step_avg:97.96ms
step:679/1770 train_time:66516ms step_avg:97.96ms
step:680/1770 train_time:66617ms step_avg:97.97ms
step:681/1770 train_time:66719ms step_avg:97.97ms
step:682/1770 train_time:66820ms step_avg:97.98ms
step:683/1770 train_time:66921ms step_avg:97.98ms
step:684/1770 train_time:67021ms step_avg:97.98ms
step:685/1770 train_time:67122ms step_avg:97.99ms
step:686/1770 train_time:67223ms step_avg:97.99ms
step:687/1770 train_time:67325ms step_avg:98.00ms
step:688/1770 train_time:67426ms step_avg:98.00ms
step:689/1770 train_time:67528ms step_avg:98.01ms
step:690/1770 train_time:67631ms step_avg:98.02ms
step:691/1770 train_time:67733ms step_avg:98.02ms
step:692/1770 train_time:67834ms step_avg:98.03ms
step:693/1770 train_time:67934ms step_avg:98.03ms
step:694/1770 train_time:68036ms step_avg:98.03ms
step:695/1770 train_time:68137ms step_avg:98.04ms
step:696/1770 train_time:68237ms step_avg:98.04ms
step:697/1770 train_time:68338ms step_avg:98.05ms
step:698/1770 train_time:68440ms step_avg:98.05ms
step:699/1770 train_time:68541ms step_avg:98.06ms
step:700/1770 train_time:68643ms step_avg:98.06ms
step:701/1770 train_time:68744ms step_avg:98.06ms
step:702/1770 train_time:68845ms step_avg:98.07ms
step:703/1770 train_time:68946ms step_avg:98.07ms
step:704/1770 train_time:69048ms step_avg:98.08ms
step:705/1770 train_time:69151ms step_avg:98.09ms
step:706/1770 train_time:69252ms step_avg:98.09ms
step:707/1770 train_time:69353ms step_avg:98.10ms
step:708/1770 train_time:69454ms step_avg:98.10ms
step:709/1770 train_time:69555ms step_avg:98.10ms
step:710/1770 train_time:69656ms step_avg:98.11ms
step:711/1770 train_time:69757ms step_avg:98.11ms
step:712/1770 train_time:69859ms step_avg:98.12ms
step:713/1770 train_time:69961ms step_avg:98.12ms
step:714/1770 train_time:70062ms step_avg:98.13ms
step:715/1770 train_time:70163ms step_avg:98.13ms
step:716/1770 train_time:70264ms step_avg:98.13ms
step:717/1770 train_time:70365ms step_avg:98.14ms
step:718/1770 train_time:70467ms step_avg:98.14ms
step:719/1770 train_time:70569ms step_avg:98.15ms
step:720/1770 train_time:70671ms step_avg:98.15ms
step:721/1770 train_time:70772ms step_avg:98.16ms
step:722/1770 train_time:70873ms step_avg:98.16ms
step:723/1770 train_time:70974ms step_avg:98.17ms
step:724/1770 train_time:71075ms step_avg:98.17ms
step:725/1770 train_time:71176ms step_avg:98.17ms
step:726/1770 train_time:71277ms step_avg:98.18ms
step:727/1770 train_time:71379ms step_avg:98.18ms
step:728/1770 train_time:71479ms step_avg:98.19ms
step:729/1770 train_time:71579ms step_avg:98.19ms
step:730/1770 train_time:71680ms step_avg:98.19ms
step:731/1770 train_time:71780ms step_avg:98.19ms
step:732/1770 train_time:71882ms step_avg:98.20ms
step:733/1770 train_time:71983ms step_avg:98.20ms
step:734/1770 train_time:72084ms step_avg:98.21ms
step:735/1770 train_time:72186ms step_avg:98.21ms
step:736/1770 train_time:72288ms step_avg:98.22ms
step:737/1770 train_time:72390ms step_avg:98.22ms
step:738/1770 train_time:72492ms step_avg:98.23ms
step:739/1770 train_time:72592ms step_avg:98.23ms
step:740/1770 train_time:72693ms step_avg:98.23ms
step:741/1770 train_time:72794ms step_avg:98.24ms
step:742/1770 train_time:72896ms step_avg:98.24ms
step:743/1770 train_time:72997ms step_avg:98.25ms
step:744/1770 train_time:73099ms step_avg:98.25ms
step:745/1770 train_time:73200ms step_avg:98.26ms
step:746/1770 train_time:73301ms step_avg:98.26ms
step:747/1770 train_time:73402ms step_avg:98.26ms
step:748/1770 train_time:73503ms step_avg:98.27ms
step:749/1770 train_time:73603ms step_avg:98.27ms
step:750/1770 train_time:73705ms step_avg:98.27ms
step:750/1770 val_loss:3.6030 train_time:73805ms step_avg:98.41ms
step:751/1770 train_time:73826ms step_avg:98.30ms
step:752/1770 train_time:73912ms step_avg:98.29ms
step:753/1770 train_time:74014ms step_avg:98.29ms
step:754/1770 train_time:74114ms step_avg:98.29ms
step:755/1770 train_time:74215ms step_avg:98.30ms
step:756/1770 train_time:74315ms step_avg:98.30ms
step:757/1770 train_time:74415ms step_avg:98.30ms
step:758/1770 train_time:74515ms step_avg:98.31ms
step:759/1770 train_time:74615ms step_avg:98.31ms
step:760/1770 train_time:74717ms step_avg:98.31ms
step:761/1770 train_time:74818ms step_avg:98.32ms
step:762/1770 train_time:74920ms step_avg:98.32ms
step:763/1770 train_time:75022ms step_avg:98.32ms
step:764/1770 train_time:75122ms step_avg:98.33ms
step:765/1770 train_time:75225ms step_avg:98.33ms
step:766/1770 train_time:75326ms step_avg:98.34ms
step:767/1770 train_time:75427ms step_avg:98.34ms
step:768/1770 train_time:75529ms step_avg:98.34ms
step:769/1770 train_time:75630ms step_avg:98.35ms
step:770/1770 train_time:75731ms step_avg:98.35ms
step:771/1770 train_time:75831ms step_avg:98.35ms
step:772/1770 train_time:75933ms step_avg:98.36ms
step:773/1770 train_time:76034ms step_avg:98.36ms
step:774/1770 train_time:76136ms step_avg:98.37ms
step:775/1770 train_time:76237ms step_avg:98.37ms
step:776/1770 train_time:76337ms step_avg:98.37ms
step:777/1770 train_time:76438ms step_avg:98.38ms
step:778/1770 train_time:76539ms step_avg:98.38ms
step:779/1770 train_time:76640ms step_avg:98.38ms
step:780/1770 train_time:76741ms step_avg:98.39ms
step:781/1770 train_time:76842ms step_avg:98.39ms
step:782/1770 train_time:76942ms step_avg:98.39ms
step:783/1770 train_time:77044ms step_avg:98.40ms
step:784/1770 train_time:77147ms step_avg:98.40ms
step:785/1770 train_time:77249ms step_avg:98.41ms
step:786/1770 train_time:77350ms step_avg:98.41ms
step:787/1770 train_time:77451ms step_avg:98.41ms
step:788/1770 train_time:77553ms step_avg:98.42ms
step:789/1770 train_time:77654ms step_avg:98.42ms
step:790/1770 train_time:77756ms step_avg:98.43ms
step:791/1770 train_time:77857ms step_avg:98.43ms
step:792/1770 train_time:77959ms step_avg:98.43ms
step:793/1770 train_time:78060ms step_avg:98.44ms
step:794/1770 train_time:78162ms step_avg:98.44ms
step:795/1770 train_time:78263ms step_avg:98.44ms
step:796/1770 train_time:78364ms step_avg:98.45ms
step:797/1770 train_time:78466ms step_avg:98.45ms
step:798/1770 train_time:78568ms step_avg:98.46ms
step:799/1770 train_time:78670ms step_avg:98.46ms
step:800/1770 train_time:78771ms step_avg:98.46ms
step:801/1770 train_time:78872ms step_avg:98.47ms
step:802/1770 train_time:78974ms step_avg:98.47ms
step:803/1770 train_time:79076ms step_avg:98.48ms
step:804/1770 train_time:79177ms step_avg:98.48ms
step:805/1770 train_time:79278ms step_avg:98.48ms
step:806/1770 train_time:79379ms step_avg:98.49ms
step:807/1770 train_time:79480ms step_avg:98.49ms
step:808/1770 train_time:79581ms step_avg:98.49ms
step:809/1770 train_time:79684ms step_avg:98.50ms
step:810/1770 train_time:79786ms step_avg:98.50ms
step:811/1770 train_time:79889ms step_avg:98.51ms
step:812/1770 train_time:79989ms step_avg:98.51ms
step:813/1770 train_time:80090ms step_avg:98.51ms
step:814/1770 train_time:80191ms step_avg:98.52ms
step:815/1770 train_time:80292ms step_avg:98.52ms
step:816/1770 train_time:80394ms step_avg:98.52ms
step:817/1770 train_time:80495ms step_avg:98.53ms
step:818/1770 train_time:80597ms step_avg:98.53ms
step:819/1770 train_time:80698ms step_avg:98.53ms
step:820/1770 train_time:80800ms step_avg:98.54ms
step:821/1770 train_time:80901ms step_avg:98.54ms
step:822/1770 train_time:81002ms step_avg:98.54ms
step:823/1770 train_time:81102ms step_avg:98.54ms
step:824/1770 train_time:81204ms step_avg:98.55ms
step:825/1770 train_time:81307ms step_avg:98.55ms
step:826/1770 train_time:81409ms step_avg:98.56ms
step:827/1770 train_time:81510ms step_avg:98.56ms
step:828/1770 train_time:81612ms step_avg:98.56ms
step:829/1770 train_time:81712ms step_avg:98.57ms
step:830/1770 train_time:81813ms step_avg:98.57ms
step:831/1770 train_time:81916ms step_avg:98.57ms
step:832/1770 train_time:82017ms step_avg:98.58ms
step:833/1770 train_time:82118ms step_avg:98.58ms
step:834/1770 train_time:82219ms step_avg:98.58ms
step:835/1770 train_time:82320ms step_avg:98.59ms
step:836/1770 train_time:82421ms step_avg:98.59ms
step:837/1770 train_time:82524ms step_avg:98.59ms
step:838/1770 train_time:82626ms step_avg:98.60ms
step:839/1770 train_time:82728ms step_avg:98.60ms
step:840/1770 train_time:82830ms step_avg:98.61ms
step:841/1770 train_time:82931ms step_avg:98.61ms
step:842/1770 train_time:83032ms step_avg:98.61ms
step:843/1770 train_time:83132ms step_avg:98.61ms
step:844/1770 train_time:83234ms step_avg:98.62ms
step:845/1770 train_time:83335ms step_avg:98.62ms
step:846/1770 train_time:83436ms step_avg:98.62ms
step:847/1770 train_time:83538ms step_avg:98.63ms
step:848/1770 train_time:83639ms step_avg:98.63ms
step:849/1770 train_time:83740ms step_avg:98.63ms
step:850/1770 train_time:83841ms step_avg:98.64ms
step:851/1770 train_time:83942ms step_avg:98.64ms
step:852/1770 train_time:84044ms step_avg:98.64ms
step:853/1770 train_time:84146ms step_avg:98.65ms
step:854/1770 train_time:84248ms step_avg:98.65ms
step:855/1770 train_time:84349ms step_avg:98.65ms
step:856/1770 train_time:84450ms step_avg:98.66ms
step:857/1770 train_time:84551ms step_avg:98.66ms
step:858/1770 train_time:84653ms step_avg:98.66ms
step:859/1770 train_time:84755ms step_avg:98.67ms
step:860/1770 train_time:84858ms step_avg:98.67ms
step:861/1770 train_time:84960ms step_avg:98.68ms
step:862/1770 train_time:85061ms step_avg:98.68ms
step:863/1770 train_time:85162ms step_avg:98.68ms
step:864/1770 train_time:85262ms step_avg:98.68ms
step:865/1770 train_time:85364ms step_avg:98.69ms
step:866/1770 train_time:85467ms step_avg:98.69ms
step:867/1770 train_time:85569ms step_avg:98.70ms
step:868/1770 train_time:85671ms step_avg:98.70ms
step:869/1770 train_time:85773ms step_avg:98.70ms
step:870/1770 train_time:85874ms step_avg:98.71ms
step:871/1770 train_time:85976ms step_avg:98.71ms
step:872/1770 train_time:86077ms step_avg:98.71ms
step:873/1770 train_time:86178ms step_avg:98.72ms
step:874/1770 train_time:86280ms step_avg:98.72ms
step:875/1770 train_time:86380ms step_avg:98.72ms
step:875/1770 val_loss:3.5506 train_time:86479ms step_avg:98.83ms
step:876/1770 train_time:86500ms step_avg:98.74ms
step:877/1770 train_time:86587ms step_avg:98.73ms
step:878/1770 train_time:86689ms step_avg:98.73ms
step:879/1770 train_time:86791ms step_avg:98.74ms
step:880/1770 train_time:86891ms step_avg:98.74ms
step:881/1770 train_time:86993ms step_avg:98.74ms
step:882/1770 train_time:87092ms step_avg:98.74ms
step:883/1770 train_time:87193ms step_avg:98.75ms
step:884/1770 train_time:87294ms step_avg:98.75ms
step:885/1770 train_time:87396ms step_avg:98.75ms
step:886/1770 train_time:87498ms step_avg:98.76ms
step:887/1770 train_time:87600ms step_avg:98.76ms
step:888/1770 train_time:87701ms step_avg:98.76ms
step:889/1770 train_time:87803ms step_avg:98.77ms
step:890/1770 train_time:87903ms step_avg:98.77ms
step:891/1770 train_time:88005ms step_avg:98.77ms
step:892/1770 train_time:88107ms step_avg:98.77ms
step:893/1770 train_time:88208ms step_avg:98.78ms
step:894/1770 train_time:88310ms step_avg:98.78ms
step:895/1770 train_time:88412ms step_avg:98.78ms
step:896/1770 train_time:88513ms step_avg:98.79ms
step:897/1770 train_time:88614ms step_avg:98.79ms
step:898/1770 train_time:88716ms step_avg:98.79ms
step:899/1770 train_time:88817ms step_avg:98.80ms
step:900/1770 train_time:88919ms step_avg:98.80ms
step:901/1770 train_time:89020ms step_avg:98.80ms
step:902/1770 train_time:89121ms step_avg:98.80ms
step:903/1770 train_time:89222ms step_avg:98.81ms
step:904/1770 train_time:89323ms step_avg:98.81ms
step:905/1770 train_time:89425ms step_avg:98.81ms
step:906/1770 train_time:89526ms step_avg:98.81ms
step:907/1770 train_time:89628ms step_avg:98.82ms
step:908/1770 train_time:89730ms step_avg:98.82ms
step:909/1770 train_time:89833ms step_avg:98.83ms
step:910/1770 train_time:89934ms step_avg:98.83ms
step:911/1770 train_time:90035ms step_avg:98.83ms
step:912/1770 train_time:90137ms step_avg:98.83ms
step:913/1770 train_time:90238ms step_avg:98.84ms
step:914/1770 train_time:90340ms step_avg:98.84ms
step:915/1770 train_time:90441ms step_avg:98.84ms
step:916/1770 train_time:90541ms step_avg:98.84ms
step:917/1770 train_time:90642ms step_avg:98.85ms
step:918/1770 train_time:90743ms step_avg:98.85ms
step:919/1770 train_time:90845ms step_avg:98.85ms
step:920/1770 train_time:90950ms step_avg:98.86ms
step:921/1770 train_time:91053ms step_avg:98.86ms
step:922/1770 train_time:91155ms step_avg:98.87ms
step:923/1770 train_time:91257ms step_avg:98.87ms
step:924/1770 train_time:91359ms step_avg:98.87ms
step:925/1770 train_time:91461ms step_avg:98.88ms
step:926/1770 train_time:91563ms step_avg:98.88ms
step:927/1770 train_time:91665ms step_avg:98.88ms
step:928/1770 train_time:91767ms step_avg:98.89ms
step:929/1770 train_time:91870ms step_avg:98.89ms
step:930/1770 train_time:91972ms step_avg:98.90ms
step:931/1770 train_time:92075ms step_avg:98.90ms
step:932/1770 train_time:92178ms step_avg:98.90ms
step:933/1770 train_time:92280ms step_avg:98.91ms
step:934/1770 train_time:92382ms step_avg:98.91ms
step:935/1770 train_time:92484ms step_avg:98.91ms
step:936/1770 train_time:92587ms step_avg:98.92ms
step:937/1770 train_time:92689ms step_avg:98.92ms
step:938/1770 train_time:92791ms step_avg:98.92ms
step:939/1770 train_time:92894ms step_avg:98.93ms
step:940/1770 train_time:92996ms step_avg:98.93ms
step:941/1770 train_time:93099ms step_avg:98.94ms
step:942/1770 train_time:93201ms step_avg:98.94ms
step:943/1770 train_time:93305ms step_avg:98.94ms
step:944/1770 train_time:93407ms step_avg:98.95ms
step:945/1770 train_time:93509ms step_avg:98.95ms
step:946/1770 train_time:93612ms step_avg:98.96ms
step:947/1770 train_time:93715ms step_avg:98.96ms
step:948/1770 train_time:93818ms step_avg:98.96ms
step:949/1770 train_time:93920ms step_avg:98.97ms
step:950/1770 train_time:94023ms step_avg:98.97ms
step:951/1770 train_time:94126ms step_avg:98.98ms
step:952/1770 train_time:94230ms step_avg:98.98ms
step:953/1770 train_time:94332ms step_avg:98.98ms
step:954/1770 train_time:94434ms step_avg:98.99ms
step:955/1770 train_time:94536ms step_avg:98.99ms
step:956/1770 train_time:94638ms step_avg:98.99ms
step:957/1770 train_time:94741ms step_avg:99.00ms
step:958/1770 train_time:94844ms step_avg:99.00ms
step:959/1770 train_time:94947ms step_avg:99.01ms
step:960/1770 train_time:95050ms step_avg:99.01ms
step:961/1770 train_time:95152ms step_avg:99.01ms
step:962/1770 train_time:95256ms step_avg:99.02ms
step:963/1770 train_time:95358ms step_avg:99.02ms
step:964/1770 train_time:95461ms step_avg:99.03ms
step:965/1770 train_time:95563ms step_avg:99.03ms
step:966/1770 train_time:95666ms step_avg:99.03ms
step:967/1770 train_time:95770ms step_avg:99.04ms
step:968/1770 train_time:95873ms step_avg:99.04ms
step:969/1770 train_time:95975ms step_avg:99.05ms
step:970/1770 train_time:96077ms step_avg:99.05ms
step:971/1770 train_time:96180ms step_avg:99.05ms
step:972/1770 train_time:96282ms step_avg:99.06ms
step:973/1770 train_time:96384ms step_avg:99.06ms
step:974/1770 train_time:96487ms step_avg:99.06ms
step:975/1770 train_time:96590ms step_avg:99.07ms
step:976/1770 train_time:96692ms step_avg:99.07ms
step:977/1770 train_time:96795ms step_avg:99.07ms
step:978/1770 train_time:96897ms step_avg:99.08ms
step:979/1770 train_time:97000ms step_avg:99.08ms
step:980/1770 train_time:97102ms step_avg:99.08ms
step:981/1770 train_time:97205ms step_avg:99.09ms
step:982/1770 train_time:97308ms step_avg:99.09ms
step:983/1770 train_time:97410ms step_avg:99.09ms
step:984/1770 train_time:97513ms step_avg:99.10ms
step:985/1770 train_time:97616ms step_avg:99.10ms
step:986/1770 train_time:97719ms step_avg:99.11ms
step:987/1770 train_time:97821ms step_avg:99.11ms
step:988/1770 train_time:97923ms step_avg:99.11ms
step:989/1770 train_time:98026ms step_avg:99.12ms
step:990/1770 train_time:98128ms step_avg:99.12ms
step:991/1770 train_time:98231ms step_avg:99.12ms
step:992/1770 train_time:98333ms step_avg:99.13ms
step:993/1770 train_time:98435ms step_avg:99.13ms
step:994/1770 train_time:98538ms step_avg:99.13ms
step:995/1770 train_time:98641ms step_avg:99.14ms
step:996/1770 train_time:98742ms step_avg:99.14ms
step:997/1770 train_time:98845ms step_avg:99.14ms
step:998/1770 train_time:98949ms step_avg:99.15ms
step:999/1770 train_time:99052ms step_avg:99.15ms
step:1000/1770 train_time:99155ms step_avg:99.15ms
step:1000/1770 val_loss:3.5136 train_time:99255ms step_avg:99.26ms
step:1001/1770 train_time:99277ms step_avg:99.18ms
step:1002/1770 train_time:99366ms step_avg:99.17ms
step:1003/1770 train_time:99469ms step_avg:99.17ms
step:1004/1770 train_time:99572ms step_avg:99.17ms
step:1005/1770 train_time:99674ms step_avg:99.18ms
step:1006/1770 train_time:99775ms step_avg:99.18ms
step:1007/1770 train_time:99877ms step_avg:99.18ms
step:1008/1770 train_time:99979ms step_avg:99.19ms
step:1009/1770 train_time:100081ms step_avg:99.19ms
step:1010/1770 train_time:100184ms step_avg:99.19ms
step:1011/1770 train_time:100287ms step_avg:99.20ms
step:1012/1770 train_time:100390ms step_avg:99.20ms
step:1013/1770 train_time:100494ms step_avg:99.20ms
step:1014/1770 train_time:100597ms step_avg:99.21ms
step:1015/1770 train_time:100700ms step_avg:99.21ms
step:1016/1770 train_time:100802ms step_avg:99.21ms
step:1017/1770 train_time:100904ms step_avg:99.22ms
step:1018/1770 train_time:101006ms step_avg:99.22ms
step:1019/1770 train_time:101108ms step_avg:99.22ms
step:1020/1770 train_time:101210ms step_avg:99.23ms
step:1021/1770 train_time:101313ms step_avg:99.23ms
step:1022/1770 train_time:101416ms step_avg:99.23ms
step:1023/1770 train_time:101519ms step_avg:99.24ms
step:1024/1770 train_time:101622ms step_avg:99.24ms
step:1025/1770 train_time:101724ms step_avg:99.24ms
step:1026/1770 train_time:101827ms step_avg:99.25ms
step:1027/1770 train_time:101929ms step_avg:99.25ms
step:1028/1770 train_time:102032ms step_avg:99.25ms
step:1029/1770 train_time:102135ms step_avg:99.26ms
step:1030/1770 train_time:102237ms step_avg:99.26ms
step:1031/1770 train_time:102339ms step_avg:99.26ms
step:1032/1770 train_time:102442ms step_avg:99.27ms
step:1033/1770 train_time:102545ms step_avg:99.27ms
step:1034/1770 train_time:102647ms step_avg:99.27ms
step:1035/1770 train_time:102749ms step_avg:99.27ms
step:1036/1770 train_time:102851ms step_avg:99.28ms
step:1037/1770 train_time:102953ms step_avg:99.28ms
step:1038/1770 train_time:103056ms step_avg:99.28ms
step:1039/1770 train_time:103159ms step_avg:99.29ms
step:1040/1770 train_time:103261ms step_avg:99.29ms
step:1041/1770 train_time:103363ms step_avg:99.29ms
step:1042/1770 train_time:103466ms step_avg:99.30ms
step:1043/1770 train_time:103569ms step_avg:99.30ms
step:1044/1770 train_time:103671ms step_avg:99.30ms
step:1045/1770 train_time:103774ms step_avg:99.31ms
step:1046/1770 train_time:103877ms step_avg:99.31ms
step:1047/1770 train_time:103979ms step_avg:99.31ms
step:1048/1770 train_time:104082ms step_avg:99.31ms
step:1049/1770 train_time:104184ms step_avg:99.32ms
step:1050/1770 train_time:104286ms step_avg:99.32ms
step:1051/1770 train_time:104388ms step_avg:99.32ms
step:1052/1770 train_time:104490ms step_avg:99.33ms
step:1053/1770 train_time:104593ms step_avg:99.33ms
step:1054/1770 train_time:104697ms step_avg:99.33ms
step:1055/1770 train_time:104799ms step_avg:99.34ms
step:1056/1770 train_time:104902ms step_avg:99.34ms
step:1057/1770 train_time:105005ms step_avg:99.34ms
step:1058/1770 train_time:105108ms step_avg:99.35ms
step:1059/1770 train_time:105211ms step_avg:99.35ms
step:1060/1770 train_time:105314ms step_avg:99.35ms
step:1061/1770 train_time:105417ms step_avg:99.36ms
step:1062/1770 train_time:105520ms step_avg:99.36ms
step:1063/1770 train_time:105624ms step_avg:99.36ms
step:1064/1770 train_time:105727ms step_avg:99.37ms
step:1065/1770 train_time:105829ms step_avg:99.37ms
step:1066/1770 train_time:105932ms step_avg:99.37ms
step:1067/1770 train_time:106035ms step_avg:99.38ms
step:1068/1770 train_time:106139ms step_avg:99.38ms
step:1069/1770 train_time:106241ms step_avg:99.38ms
step:1070/1770 train_time:106344ms step_avg:99.39ms
step:1071/1770 train_time:106446ms step_avg:99.39ms
step:1072/1770 train_time:106549ms step_avg:99.39ms
step:1073/1770 train_time:106652ms step_avg:99.40ms
step:1074/1770 train_time:106755ms step_avg:99.40ms
step:1075/1770 train_time:106857ms step_avg:99.40ms
step:1076/1770 train_time:106961ms step_avg:99.41ms
step:1077/1770 train_time:107064ms step_avg:99.41ms
step:1078/1770 train_time:107166ms step_avg:99.41ms
step:1079/1770 train_time:107269ms step_avg:99.42ms
step:1080/1770 train_time:107373ms step_avg:99.42ms
step:1081/1770 train_time:107476ms step_avg:99.42ms
step:1082/1770 train_time:107578ms step_avg:99.43ms
step:1083/1770 train_time:107681ms step_avg:99.43ms
step:1084/1770 train_time:107785ms step_avg:99.43ms
step:1085/1770 train_time:107887ms step_avg:99.44ms
step:1086/1770 train_time:107990ms step_avg:99.44ms
step:1087/1770 train_time:108091ms step_avg:99.44ms
step:1088/1770 train_time:108195ms step_avg:99.44ms
step:1089/1770 train_time:108298ms step_avg:99.45ms
step:1090/1770 train_time:108401ms step_avg:99.45ms
step:1091/1770 train_time:108503ms step_avg:99.45ms
step:1092/1770 train_time:108606ms step_avg:99.46ms
step:1093/1770 train_time:108708ms step_avg:99.46ms
step:1094/1770 train_time:108810ms step_avg:99.46ms
step:1095/1770 train_time:108913ms step_avg:99.46ms
step:1096/1770 train_time:109016ms step_avg:99.47ms
step:1097/1770 train_time:109120ms step_avg:99.47ms
step:1098/1770 train_time:109222ms step_avg:99.47ms
step:1099/1770 train_time:109325ms step_avg:99.48ms
step:1100/1770 train_time:109428ms step_avg:99.48ms
step:1101/1770 train_time:109530ms step_avg:99.48ms
step:1102/1770 train_time:109633ms step_avg:99.49ms
step:1103/1770 train_time:109736ms step_avg:99.49ms
step:1104/1770 train_time:109838ms step_avg:99.49ms
step:1105/1770 train_time:109941ms step_avg:99.49ms
step:1106/1770 train_time:110043ms step_avg:99.50ms
step:1107/1770 train_time:110146ms step_avg:99.50ms
step:1108/1770 train_time:110248ms step_avg:99.50ms
step:1109/1770 train_time:110351ms step_avg:99.50ms
step:1110/1770 train_time:110454ms step_avg:99.51ms
step:1111/1770 train_time:110558ms step_avg:99.51ms
step:1112/1770 train_time:110662ms step_avg:99.52ms
step:1113/1770 train_time:110764ms step_avg:99.52ms
step:1114/1770 train_time:110868ms step_avg:99.52ms
step:1115/1770 train_time:110972ms step_avg:99.53ms
step:1116/1770 train_time:111076ms step_avg:99.53ms
step:1117/1770 train_time:111179ms step_avg:99.53ms
step:1118/1770 train_time:111281ms step_avg:99.54ms
step:1119/1770 train_time:111384ms step_avg:99.54ms
step:1120/1770 train_time:111486ms step_avg:99.54ms
step:1121/1770 train_time:111588ms step_avg:99.54ms
step:1122/1770 train_time:111692ms step_avg:99.55ms
step:1123/1770 train_time:111795ms step_avg:99.55ms
step:1124/1770 train_time:111897ms step_avg:99.55ms
step:1125/1770 train_time:112000ms step_avg:99.56ms
step:1125/1770 val_loss:3.4729 train_time:112101ms step_avg:99.65ms
step:1126/1770 train_time:112122ms step_avg:99.58ms
step:1127/1770 train_time:112214ms step_avg:99.57ms
step:1128/1770 train_time:112317ms step_avg:99.57ms
step:1129/1770 train_time:112420ms step_avg:99.57ms
step:1130/1770 train_time:112522ms step_avg:99.58ms
step:1131/1770 train_time:112625ms step_avg:99.58ms
step:1132/1770 train_time:112727ms step_avg:99.58ms
step:1133/1770 train_time:112829ms step_avg:99.58ms
step:1134/1770 train_time:112931ms step_avg:99.59ms
step:1135/1770 train_time:113033ms step_avg:99.59ms
step:1136/1770 train_time:113136ms step_avg:99.59ms
step:1137/1770 train_time:113241ms step_avg:99.60ms
step:1138/1770 train_time:113343ms step_avg:99.60ms
step:1139/1770 train_time:113447ms step_avg:99.60ms
step:1140/1770 train_time:113550ms step_avg:99.61ms
step:1141/1770 train_time:113652ms step_avg:99.61ms
step:1142/1770 train_time:113754ms step_avg:99.61ms
step:1143/1770 train_time:113856ms step_avg:99.61ms
step:1144/1770 train_time:113959ms step_avg:99.61ms
step:1145/1770 train_time:114062ms step_avg:99.62ms
step:1146/1770 train_time:114165ms step_avg:99.62ms
step:1147/1770 train_time:114268ms step_avg:99.62ms
step:1148/1770 train_time:114371ms step_avg:99.63ms
step:1149/1770 train_time:114474ms step_avg:99.63ms
step:1150/1770 train_time:114577ms step_avg:99.63ms
step:1151/1770 train_time:114679ms step_avg:99.63ms
step:1152/1770 train_time:114782ms step_avg:99.64ms
step:1153/1770 train_time:114885ms step_avg:99.64ms
step:1154/1770 train_time:114988ms step_avg:99.64ms
step:1155/1770 train_time:115090ms step_avg:99.65ms
step:1156/1770 train_time:115192ms step_avg:99.65ms
step:1157/1770 train_time:115296ms step_avg:99.65ms
step:1158/1770 train_time:115399ms step_avg:99.65ms
step:1159/1770 train_time:115502ms step_avg:99.66ms
step:1160/1770 train_time:115605ms step_avg:99.66ms
step:1161/1770 train_time:115707ms step_avg:99.66ms
step:1162/1770 train_time:115810ms step_avg:99.66ms
step:1163/1770 train_time:115913ms step_avg:99.67ms
step:1164/1770 train_time:116015ms step_avg:99.67ms
step:1165/1770 train_time:116118ms step_avg:99.67ms
step:1166/1770 train_time:116221ms step_avg:99.68ms
step:1167/1770 train_time:116324ms step_avg:99.68ms
step:1168/1770 train_time:116427ms step_avg:99.68ms
step:1169/1770 train_time:116529ms step_avg:99.68ms
step:1170/1770 train_time:116631ms step_avg:99.68ms
step:1171/1770 train_time:116734ms step_avg:99.69ms
step:1172/1770 train_time:116837ms step_avg:99.69ms
step:1173/1770 train_time:116940ms step_avg:99.69ms
step:1174/1770 train_time:117044ms step_avg:99.70ms
step:1175/1770 train_time:117146ms step_avg:99.70ms
step:1176/1770 train_time:117249ms step_avg:99.70ms
step:1177/1770 train_time:117351ms step_avg:99.70ms
step:1178/1770 train_time:117454ms step_avg:99.71ms
step:1179/1770 train_time:117556ms step_avg:99.71ms
step:1180/1770 train_time:117660ms step_avg:99.71ms
step:1181/1770 train_time:117761ms step_avg:99.71ms
step:1182/1770 train_time:117864ms step_avg:99.72ms
step:1183/1770 train_time:117968ms step_avg:99.72ms
step:1184/1770 train_time:118074ms step_avg:99.72ms
step:1185/1770 train_time:118177ms step_avg:99.73ms
step:1186/1770 train_time:118282ms step_avg:99.73ms
step:1187/1770 train_time:118387ms step_avg:99.74ms
step:1188/1770 train_time:118490ms step_avg:99.74ms
step:1189/1770 train_time:118594ms step_avg:99.74ms
step:1190/1770 train_time:118698ms step_avg:99.75ms
step:1191/1770 train_time:118803ms step_avg:99.75ms
step:1192/1770 train_time:118907ms step_avg:99.75ms
step:1193/1770 train_time:119011ms step_avg:99.76ms
step:1194/1770 train_time:119114ms step_avg:99.76ms
step:1195/1770 train_time:119218ms step_avg:99.76ms
step:1196/1770 train_time:119323ms step_avg:99.77ms
step:1197/1770 train_time:119427ms step_avg:99.77ms
step:1198/1770 train_time:119531ms step_avg:99.78ms
step:1199/1770 train_time:119635ms step_avg:99.78ms
step:1200/1770 train_time:119740ms step_avg:99.78ms
step:1201/1770 train_time:119844ms step_avg:99.79ms
step:1202/1770 train_time:119948ms step_avg:99.79ms
step:1203/1770 train_time:120052ms step_avg:99.79ms
step:1204/1770 train_time:120156ms step_avg:99.80ms
step:1205/1770 train_time:120259ms step_avg:99.80ms
step:1206/1770 train_time:120366ms step_avg:99.81ms
step:1207/1770 train_time:120470ms step_avg:99.81ms
step:1208/1770 train_time:120574ms step_avg:99.81ms
step:1209/1770 train_time:120678ms step_avg:99.82ms
step:1210/1770 train_time:120782ms step_avg:99.82ms
step:1211/1770 train_time:120887ms step_avg:99.82ms
step:1212/1770 train_time:120994ms step_avg:99.83ms
step:1213/1770 train_time:121097ms step_avg:99.83ms
step:1214/1770 train_time:121200ms step_avg:99.84ms
step:1215/1770 train_time:121305ms step_avg:99.84ms
step:1216/1770 train_time:121411ms step_avg:99.84ms
step:1217/1770 train_time:121514ms step_avg:99.85ms
step:1218/1770 train_time:121618ms step_avg:99.85ms
step:1219/1770 train_time:121722ms step_avg:99.85ms
step:1220/1770 train_time:121826ms step_avg:99.86ms
step:1221/1770 train_time:121929ms step_avg:99.86ms
step:1222/1770 train_time:122034ms step_avg:99.86ms
step:1223/1770 train_time:122137ms step_avg:99.87ms
step:1224/1770 train_time:122242ms step_avg:99.87ms
step:1225/1770 train_time:122347ms step_avg:99.88ms
step:1226/1770 train_time:122451ms step_avg:99.88ms
step:1227/1770 train_time:122556ms step_avg:99.88ms
step:1228/1770 train_time:122662ms step_avg:99.89ms
step:1229/1770 train_time:122767ms step_avg:99.89ms
step:1230/1770 train_time:122870ms step_avg:99.89ms
step:1231/1770 train_time:122974ms step_avg:99.90ms
step:1232/1770 train_time:123077ms step_avg:99.90ms
step:1233/1770 train_time:123182ms step_avg:99.90ms
step:1234/1770 train_time:123287ms step_avg:99.91ms
step:1235/1770 train_time:123391ms step_avg:99.91ms
step:1236/1770 train_time:123496ms step_avg:99.92ms
step:1237/1770 train_time:123600ms step_avg:99.92ms
step:1238/1770 train_time:123704ms step_avg:99.92ms
step:1239/1770 train_time:123808ms step_avg:99.93ms
step:1240/1770 train_time:123911ms step_avg:99.93ms
step:1241/1770 train_time:124016ms step_avg:99.93ms
step:1242/1770 train_time:124119ms step_avg:99.94ms
step:1243/1770 train_time:124223ms step_avg:99.94ms
step:1244/1770 train_time:124327ms step_avg:99.94ms
step:1245/1770 train_time:124431ms step_avg:99.94ms
step:1246/1770 train_time:124535ms step_avg:99.95ms
step:1247/1770 train_time:124639ms step_avg:99.95ms
step:1248/1770 train_time:124743ms step_avg:99.95ms
step:1249/1770 train_time:124846ms step_avg:99.96ms
step:1250/1770 train_time:124950ms step_avg:99.96ms
step:1250/1770 val_loss:3.4255 train_time:125054ms step_avg:100.04ms
step:1251/1770 train_time:125075ms step_avg:99.98ms
step:1252/1770 train_time:125165ms step_avg:99.97ms
step:1253/1770 train_time:125269ms step_avg:99.98ms
step:1254/1770 train_time:125372ms step_avg:99.98ms
step:1255/1770 train_time:125478ms step_avg:99.98ms
step:1256/1770 train_time:125582ms step_avg:99.99ms
step:1257/1770 train_time:125685ms step_avg:99.99ms
step:1258/1770 train_time:125789ms step_avg:99.99ms
step:1259/1770 train_time:125892ms step_avg:99.99ms
step:1260/1770 train_time:125996ms step_avg:100.00ms
step:1261/1770 train_time:126102ms step_avg:100.00ms
step:1262/1770 train_time:126208ms step_avg:100.01ms
step:1263/1770 train_time:126312ms step_avg:100.01ms
step:1264/1770 train_time:126417ms step_avg:100.01ms
step:1265/1770 train_time:126521ms step_avg:100.02ms
step:1266/1770 train_time:126624ms step_avg:100.02ms
step:1267/1770 train_time:126728ms step_avg:100.02ms
step:1268/1770 train_time:126832ms step_avg:100.03ms
step:1269/1770 train_time:126936ms step_avg:100.03ms
step:1270/1770 train_time:127041ms step_avg:100.03ms
step:1271/1770 train_time:127145ms step_avg:100.04ms
step:1272/1770 train_time:127249ms step_avg:100.04ms
step:1273/1770 train_time:127353ms step_avg:100.04ms
step:1274/1770 train_time:127458ms step_avg:100.05ms
step:1275/1770 train_time:127562ms step_avg:100.05ms
step:1276/1770 train_time:127666ms step_avg:100.05ms
step:1277/1770 train_time:127770ms step_avg:100.05ms
step:1278/1770 train_time:127874ms step_avg:100.06ms
step:1279/1770 train_time:127978ms step_avg:100.06ms
step:1280/1770 train_time:128082ms step_avg:100.06ms
step:1281/1770 train_time:128186ms step_avg:100.07ms
step:1282/1770 train_time:128290ms step_avg:100.07ms
step:1283/1770 train_time:128395ms step_avg:100.07ms
step:1284/1770 train_time:128500ms step_avg:100.08ms
step:1285/1770 train_time:128604ms step_avg:100.08ms
step:1286/1770 train_time:128708ms step_avg:100.08ms
step:1287/1770 train_time:128813ms step_avg:100.09ms
step:1288/1770 train_time:128917ms step_avg:100.09ms
step:1289/1770 train_time:129020ms step_avg:100.09ms
step:1290/1770 train_time:129124ms step_avg:100.10ms
step:1291/1770 train_time:129227ms step_avg:100.10ms
step:1292/1770 train_time:129330ms step_avg:100.10ms
step:1293/1770 train_time:129435ms step_avg:100.10ms
step:1294/1770 train_time:129539ms step_avg:100.11ms
step:1295/1770 train_time:129644ms step_avg:100.11ms
step:1296/1770 train_time:129748ms step_avg:100.11ms
step:1297/1770 train_time:129851ms step_avg:100.12ms
step:1298/1770 train_time:129956ms step_avg:100.12ms
step:1299/1770 train_time:130060ms step_avg:100.12ms
step:1300/1770 train_time:130164ms step_avg:100.13ms
step:1301/1770 train_time:130269ms step_avg:100.13ms
step:1302/1770 train_time:130373ms step_avg:100.13ms
step:1303/1770 train_time:130477ms step_avg:100.14ms
step:1304/1770 train_time:130581ms step_avg:100.14ms
step:1305/1770 train_time:130685ms step_avg:100.14ms
step:1306/1770 train_time:130789ms step_avg:100.14ms
step:1307/1770 train_time:130892ms step_avg:100.15ms
step:1308/1770 train_time:130996ms step_avg:100.15ms
step:1309/1770 train_time:131101ms step_avg:100.15ms
step:1310/1770 train_time:131205ms step_avg:100.16ms
step:1311/1770 train_time:131308ms step_avg:100.16ms
step:1312/1770 train_time:131411ms step_avg:100.16ms
step:1313/1770 train_time:131515ms step_avg:100.16ms
step:1314/1770 train_time:131620ms step_avg:100.17ms
step:1315/1770 train_time:131723ms step_avg:100.17ms
step:1316/1770 train_time:131826ms step_avg:100.17ms
step:1317/1770 train_time:131930ms step_avg:100.17ms
step:1318/1770 train_time:132036ms step_avg:100.18ms
step:1319/1770 train_time:132140ms step_avg:100.18ms
step:1320/1770 train_time:132244ms step_avg:100.18ms
step:1321/1770 train_time:132348ms step_avg:100.19ms
step:1322/1770 train_time:132452ms step_avg:100.19ms
step:1323/1770 train_time:132556ms step_avg:100.19ms
step:1324/1770 train_time:132660ms step_avg:100.20ms
step:1325/1770 train_time:132765ms step_avg:100.20ms
step:1326/1770 train_time:132868ms step_avg:100.20ms
step:1327/1770 train_time:132975ms step_avg:100.21ms
step:1328/1770 train_time:133079ms step_avg:100.21ms
step:1329/1770 train_time:133184ms step_avg:100.21ms
step:1330/1770 train_time:133287ms step_avg:100.22ms
step:1331/1770 train_time:133391ms step_avg:100.22ms
step:1332/1770 train_time:133494ms step_avg:100.22ms
step:1333/1770 train_time:133598ms step_avg:100.22ms
step:1334/1770 train_time:133701ms step_avg:100.23ms
step:1335/1770 train_time:133805ms step_avg:100.23ms
step:1336/1770 train_time:133909ms step_avg:100.23ms
step:1337/1770 train_time:134014ms step_avg:100.23ms
step:1338/1770 train_time:134118ms step_avg:100.24ms
step:1339/1770 train_time:134223ms step_avg:100.24ms
step:1340/1770 train_time:134328ms step_avg:100.24ms
step:1341/1770 train_time:134432ms step_avg:100.25ms
step:1342/1770 train_time:134536ms step_avg:100.25ms
step:1343/1770 train_time:134642ms step_avg:100.25ms
step:1344/1770 train_time:134746ms step_avg:100.26ms
step:1345/1770 train_time:134850ms step_avg:100.26ms
step:1346/1770 train_time:134955ms step_avg:100.26ms
step:1347/1770 train_time:135060ms step_avg:100.27ms
step:1348/1770 train_time:135166ms step_avg:100.27ms
step:1349/1770 train_time:135271ms step_avg:100.27ms
step:1350/1770 train_time:135374ms step_avg:100.28ms
step:1351/1770 train_time:135479ms step_avg:100.28ms
step:1352/1770 train_time:135583ms step_avg:100.28ms
step:1353/1770 train_time:135688ms step_avg:100.29ms
step:1354/1770 train_time:135791ms step_avg:100.29ms
step:1355/1770 train_time:135894ms step_avg:100.29ms
step:1356/1770 train_time:135999ms step_avg:100.29ms
step:1357/1770 train_time:136103ms step_avg:100.30ms
step:1358/1770 train_time:136209ms step_avg:100.30ms
step:1359/1770 train_time:136313ms step_avg:100.30ms
step:1360/1770 train_time:136418ms step_avg:100.31ms
step:1361/1770 train_time:136522ms step_avg:100.31ms
step:1362/1770 train_time:136626ms step_avg:100.31ms
step:1363/1770 train_time:136731ms step_avg:100.32ms
step:1364/1770 train_time:136834ms step_avg:100.32ms
step:1365/1770 train_time:136939ms step_avg:100.32ms
step:1366/1770 train_time:137043ms step_avg:100.32ms
step:1367/1770 train_time:137147ms step_avg:100.33ms
step:1368/1770 train_time:137250ms step_avg:100.33ms
step:1369/1770 train_time:137356ms step_avg:100.33ms
step:1370/1770 train_time:137462ms step_avg:100.34ms
step:1371/1770 train_time:137566ms step_avg:100.34ms
step:1372/1770 train_time:137669ms step_avg:100.34ms
step:1373/1770 train_time:137773ms step_avg:100.34ms
step:1374/1770 train_time:137879ms step_avg:100.35ms
step:1375/1770 train_time:137983ms step_avg:100.35ms
step:1375/1770 val_loss:3.3806 train_time:138086ms step_avg:100.43ms
step:1376/1770 train_time:138107ms step_avg:100.37ms
step:1377/1770 train_time:138198ms step_avg:100.36ms
step:1378/1770 train_time:138302ms step_avg:100.36ms
step:1379/1770 train_time:138405ms step_avg:100.37ms
step:1380/1770 train_time:138509ms step_avg:100.37ms
step:1381/1770 train_time:138613ms step_avg:100.37ms
step:1382/1770 train_time:138716ms step_avg:100.37ms
step:1383/1770 train_time:138821ms step_avg:100.38ms
step:1384/1770 train_time:138925ms step_avg:100.38ms
step:1385/1770 train_time:139029ms step_avg:100.38ms
step:1386/1770 train_time:139136ms step_avg:100.39ms
step:1387/1770 train_time:139240ms step_avg:100.39ms
step:1388/1770 train_time:139343ms step_avg:100.39ms
step:1389/1770 train_time:139446ms step_avg:100.39ms
step:1390/1770 train_time:139550ms step_avg:100.40ms
step:1391/1770 train_time:139654ms step_avg:100.40ms
step:1392/1770 train_time:139758ms step_avg:100.40ms
step:1393/1770 train_time:139862ms step_avg:100.40ms
step:1394/1770 train_time:139965ms step_avg:100.41ms
step:1395/1770 train_time:140071ms step_avg:100.41ms
step:1396/1770 train_time:140176ms step_avg:100.41ms
step:1397/1770 train_time:140281ms step_avg:100.42ms
step:1398/1770 train_time:140385ms step_avg:100.42ms
step:1399/1770 train_time:140489ms step_avg:100.42ms
step:1400/1770 train_time:140593ms step_avg:100.42ms
step:1401/1770 train_time:140698ms step_avg:100.43ms
step:1402/1770 train_time:140802ms step_avg:100.43ms
step:1403/1770 train_time:140906ms step_avg:100.43ms
step:1404/1770 train_time:141010ms step_avg:100.43ms
step:1405/1770 train_time:141114ms step_avg:100.44ms
step:1406/1770 train_time:141219ms step_avg:100.44ms
step:1407/1770 train_time:141323ms step_avg:100.44ms
step:1408/1770 train_time:141427ms step_avg:100.45ms
step:1409/1770 train_time:141531ms step_avg:100.45ms
step:1410/1770 train_time:141635ms step_avg:100.45ms
step:1411/1770 train_time:141740ms step_avg:100.45ms
step:1412/1770 train_time:141843ms step_avg:100.46ms
step:1413/1770 train_time:141946ms step_avg:100.46ms
step:1414/1770 train_time:142052ms step_avg:100.46ms
step:1415/1770 train_time:142157ms step_avg:100.46ms
step:1416/1770 train_time:142263ms step_avg:100.47ms
step:1417/1770 train_time:142367ms step_avg:100.47ms
step:1418/1770 train_time:142471ms step_avg:100.47ms
step:1419/1770 train_time:142576ms step_avg:100.48ms
step:1420/1770 train_time:142680ms step_avg:100.48ms
step:1421/1770 train_time:142784ms step_avg:100.48ms
step:1422/1770 train_time:142888ms step_avg:100.48ms
step:1423/1770 train_time:142992ms step_avg:100.49ms
step:1424/1770 train_time:143097ms step_avg:100.49ms
step:1425/1770 train_time:143200ms step_avg:100.49ms
step:1426/1770 train_time:143306ms step_avg:100.49ms
step:1427/1770 train_time:143409ms step_avg:100.50ms
step:1428/1770 train_time:143516ms step_avg:100.50ms
step:1429/1770 train_time:143620ms step_avg:100.50ms
step:1430/1770 train_time:143724ms step_avg:100.51ms
step:1431/1770 train_time:143829ms step_avg:100.51ms
step:1432/1770 train_time:143933ms step_avg:100.51ms
step:1433/1770 train_time:144036ms step_avg:100.51ms
step:1434/1770 train_time:144140ms step_avg:100.52ms
step:1435/1770 train_time:144244ms step_avg:100.52ms
step:1436/1770 train_time:144351ms step_avg:100.52ms
step:1437/1770 train_time:144455ms step_avg:100.53ms
step:1438/1770 train_time:144559ms step_avg:100.53ms
step:1439/1770 train_time:144664ms step_avg:100.53ms
step:1440/1770 train_time:144767ms step_avg:100.53ms
step:1441/1770 train_time:144874ms step_avg:100.54ms
step:1442/1770 train_time:144977ms step_avg:100.54ms
step:1443/1770 train_time:145081ms step_avg:100.54ms
step:1444/1770 train_time:145186ms step_avg:100.54ms
step:1445/1770 train_time:145290ms step_avg:100.55ms
step:1446/1770 train_time:145397ms step_avg:100.55ms
step:1447/1770 train_time:145502ms step_avg:100.55ms
step:1448/1770 train_time:145607ms step_avg:100.56ms
step:1449/1770 train_time:145714ms step_avg:100.56ms
step:1450/1770 train_time:145819ms step_avg:100.56ms
step:1451/1770 train_time:145924ms step_avg:100.57ms
step:1452/1770 train_time:146030ms step_avg:100.57ms
step:1453/1770 train_time:146134ms step_avg:100.57ms
step:1454/1770 train_time:146239ms step_avg:100.58ms
step:1455/1770 train_time:146345ms step_avg:100.58ms
step:1456/1770 train_time:146452ms step_avg:100.59ms
step:1457/1770 train_time:146558ms step_avg:100.59ms
step:1458/1770 train_time:146663ms step_avg:100.59ms
step:1459/1770 train_time:146769ms step_avg:100.60ms
step:1460/1770 train_time:146873ms step_avg:100.60ms
step:1461/1770 train_time:146978ms step_avg:100.60ms
step:1462/1770 train_time:147083ms step_avg:100.60ms
step:1463/1770 train_time:147187ms step_avg:100.61ms
step:1464/1770 train_time:147295ms step_avg:100.61ms
step:1465/1770 train_time:147400ms step_avg:100.61ms
step:1466/1770 train_time:147505ms step_avg:100.62ms
step:1467/1770 train_time:147610ms step_avg:100.62ms
step:1468/1770 train_time:147717ms step_avg:100.62ms
step:1469/1770 train_time:147822ms step_avg:100.63ms
step:1470/1770 train_time:147926ms step_avg:100.63ms
step:1471/1770 train_time:148032ms step_avg:100.63ms
step:1472/1770 train_time:148137ms step_avg:100.64ms
step:1473/1770 train_time:148243ms step_avg:100.64ms
step:1474/1770 train_time:148348ms step_avg:100.64ms
step:1475/1770 train_time:148455ms step_avg:100.65ms
step:1476/1770 train_time:148560ms step_avg:100.65ms
step:1477/1770 train_time:148667ms step_avg:100.65ms
step:1478/1770 train_time:148774ms step_avg:100.66ms
step:1479/1770 train_time:148880ms step_avg:100.66ms
step:1480/1770 train_time:148985ms step_avg:100.67ms
step:1481/1770 train_time:149093ms step_avg:100.67ms
step:1482/1770 train_time:149197ms step_avg:100.67ms
step:1483/1770 train_time:149303ms step_avg:100.68ms
step:1484/1770 train_time:149408ms step_avg:100.68ms
step:1485/1770 train_time:149513ms step_avg:100.68ms
step:1486/1770 train_time:149618ms step_avg:100.69ms
step:1487/1770 train_time:149723ms step_avg:100.69ms
step:1488/1770 train_time:149828ms step_avg:100.69ms
step:1489/1770 train_time:149934ms step_avg:100.69ms
step:1490/1770 train_time:150039ms step_avg:100.70ms
step:1491/1770 train_time:150144ms step_avg:100.70ms
step:1492/1770 train_time:150251ms step_avg:100.70ms
step:1493/1770 train_time:150360ms step_avg:100.71ms
step:1494/1770 train_time:150467ms step_avg:100.71ms
step:1495/1770 train_time:150573ms step_avg:100.72ms
step:1496/1770 train_time:150678ms step_avg:100.72ms
step:1497/1770 train_time:150783ms step_avg:100.72ms
step:1498/1770 train_time:150887ms step_avg:100.73ms
step:1499/1770 train_time:150992ms step_avg:100.73ms
step:1500/1770 train_time:151097ms step_avg:100.73ms
step:1500/1770 val_loss:3.3425 train_time:151200ms step_avg:100.80ms
step:1501/1770 train_time:151222ms step_avg:100.75ms
step:1502/1770 train_time:151317ms step_avg:100.74ms
step:1503/1770 train_time:151421ms step_avg:100.75ms
step:1504/1770 train_time:151527ms step_avg:100.75ms
step:1505/1770 train_time:151634ms step_avg:100.75ms
step:1506/1770 train_time:151740ms step_avg:100.76ms
step:1507/1770 train_time:151845ms step_avg:100.76ms
step:1508/1770 train_time:151952ms step_avg:100.76ms
step:1509/1770 train_time:152057ms step_avg:100.77ms
step:1510/1770 train_time:152161ms step_avg:100.77ms
step:1511/1770 train_time:152268ms step_avg:100.77ms
step:1512/1770 train_time:152373ms step_avg:100.78ms
step:1513/1770 train_time:152478ms step_avg:100.78ms
step:1514/1770 train_time:152583ms step_avg:100.78ms
step:1515/1770 train_time:152688ms step_avg:100.78ms
step:1516/1770 train_time:152793ms step_avg:100.79ms
step:1517/1770 train_time:152900ms step_avg:100.79ms
step:1518/1770 train_time:153006ms step_avg:100.79ms
step:1519/1770 train_time:153110ms step_avg:100.80ms
step:1520/1770 train_time:153216ms step_avg:100.80ms
step:1521/1770 train_time:153321ms step_avg:100.80ms
step:1522/1770 train_time:153427ms step_avg:100.81ms
step:1523/1770 train_time:153533ms step_avg:100.81ms
step:1524/1770 train_time:153638ms step_avg:100.81ms
step:1525/1770 train_time:153743ms step_avg:100.81ms
step:1526/1770 train_time:153848ms step_avg:100.82ms
step:1527/1770 train_time:153953ms step_avg:100.82ms
step:1528/1770 train_time:154060ms step_avg:100.82ms
step:1529/1770 train_time:154166ms step_avg:100.83ms
step:1530/1770 train_time:154271ms step_avg:100.83ms
step:1531/1770 train_time:154377ms step_avg:100.83ms
step:1532/1770 train_time:154484ms step_avg:100.84ms
step:1533/1770 train_time:154589ms step_avg:100.84ms
step:1534/1770 train_time:154696ms step_avg:100.84ms
step:1535/1770 train_time:154800ms step_avg:100.85ms
step:1536/1770 train_time:154906ms step_avg:100.85ms
step:1537/1770 train_time:155011ms step_avg:100.85ms
step:1538/1770 train_time:155117ms step_avg:100.86ms
step:1539/1770 train_time:155223ms step_avg:100.86ms
step:1540/1770 train_time:155330ms step_avg:100.86ms
step:1541/1770 train_time:155436ms step_avg:100.87ms
step:1542/1770 train_time:155541ms step_avg:100.87ms
step:1543/1770 train_time:155646ms step_avg:100.87ms
step:1544/1770 train_time:155754ms step_avg:100.88ms
step:1545/1770 train_time:155859ms step_avg:100.88ms
step:1546/1770 train_time:155965ms step_avg:100.88ms
step:1547/1770 train_time:156069ms step_avg:100.89ms
step:1548/1770 train_time:156174ms step_avg:100.89ms
step:1549/1770 train_time:156279ms step_avg:100.89ms
step:1550/1770 train_time:156384ms step_avg:100.89ms
step:1551/1770 train_time:156489ms step_avg:100.90ms
step:1552/1770 train_time:156597ms step_avg:100.90ms
step:1553/1770 train_time:156702ms step_avg:100.90ms
step:1554/1770 train_time:156807ms step_avg:100.91ms
step:1555/1770 train_time:156913ms step_avg:100.91ms
step:1556/1770 train_time:157017ms step_avg:100.91ms
step:1557/1770 train_time:157123ms step_avg:100.91ms
step:1558/1770 train_time:157227ms step_avg:100.92ms
step:1559/1770 train_time:157332ms step_avg:100.92ms
step:1560/1770 train_time:157438ms step_avg:100.92ms
step:1561/1770 train_time:157546ms step_avg:100.93ms
step:1562/1770 train_time:157651ms step_avg:100.93ms
step:1563/1770 train_time:157756ms step_avg:100.93ms
step:1564/1770 train_time:157861ms step_avg:100.93ms
step:1565/1770 train_time:157966ms step_avg:100.94ms
step:1566/1770 train_time:158070ms step_avg:100.94ms
step:1567/1770 train_time:158176ms step_avg:100.94ms
step:1568/1770 train_time:158281ms step_avg:100.94ms
step:1569/1770 train_time:158389ms step_avg:100.95ms
step:1570/1770 train_time:158495ms step_avg:100.95ms
step:1571/1770 train_time:158600ms step_avg:100.95ms
step:1572/1770 train_time:158706ms step_avg:100.96ms
step:1573/1770 train_time:158814ms step_avg:100.96ms
step:1574/1770 train_time:158918ms step_avg:100.96ms
step:1575/1770 train_time:159023ms step_avg:100.97ms
step:1576/1770 train_time:159127ms step_avg:100.97ms
step:1577/1770 train_time:159234ms step_avg:100.97ms
step:1578/1770 train_time:159341ms step_avg:100.98ms
step:1579/1770 train_time:159447ms step_avg:100.98ms
step:1580/1770 train_time:159552ms step_avg:100.98ms
step:1581/1770 train_time:159659ms step_avg:100.99ms
step:1582/1770 train_time:159765ms step_avg:100.99ms
step:1583/1770 train_time:159870ms step_avg:100.99ms
step:1584/1770 train_time:159976ms step_avg:101.00ms
step:1585/1770 train_time:160081ms step_avg:101.00ms
step:1586/1770 train_time:160190ms step_avg:101.00ms
step:1587/1770 train_time:160296ms step_avg:101.01ms
step:1588/1770 train_time:160401ms step_avg:101.01ms
step:1589/1770 train_time:160509ms step_avg:101.01ms
step:1590/1770 train_time:160613ms step_avg:101.01ms
step:1591/1770 train_time:160717ms step_avg:101.02ms
step:1592/1770 train_time:160823ms step_avg:101.02ms
step:1593/1770 train_time:160928ms step_avg:101.02ms
step:1594/1770 train_time:161033ms step_avg:101.02ms
step:1595/1770 train_time:161139ms step_avg:101.03ms
step:1596/1770 train_time:161244ms step_avg:101.03ms
step:1597/1770 train_time:161349ms step_avg:101.03ms
step:1598/1770 train_time:161455ms step_avg:101.04ms
step:1599/1770 train_time:161562ms step_avg:101.04ms
step:1600/1770 train_time:161669ms step_avg:101.04ms
step:1601/1770 train_time:161775ms step_avg:101.05ms
step:1602/1770 train_time:161881ms step_avg:101.05ms
step:1603/1770 train_time:161986ms step_avg:101.05ms
step:1604/1770 train_time:162091ms step_avg:101.05ms
step:1605/1770 train_time:162195ms step_avg:101.06ms
step:1606/1770 train_time:162302ms step_avg:101.06ms
step:1607/1770 train_time:162411ms step_avg:101.06ms
step:1608/1770 train_time:162516ms step_avg:101.07ms
step:1609/1770 train_time:162622ms step_avg:101.07ms
step:1610/1770 train_time:162729ms step_avg:101.07ms
step:1611/1770 train_time:162836ms step_avg:101.08ms
step:1612/1770 train_time:162941ms step_avg:101.08ms
step:1613/1770 train_time:163047ms step_avg:101.08ms
step:1614/1770 train_time:163152ms step_avg:101.09ms
step:1615/1770 train_time:163258ms step_avg:101.09ms
step:1616/1770 train_time:163363ms step_avg:101.09ms
step:1617/1770 train_time:163470ms step_avg:101.09ms
step:1618/1770 train_time:163576ms step_avg:101.10ms
step:1619/1770 train_time:163684ms step_avg:101.10ms
step:1620/1770 train_time:163791ms step_avg:101.11ms
step:1621/1770 train_time:163896ms step_avg:101.11ms
step:1622/1770 train_time:164002ms step_avg:101.11ms
step:1623/1770 train_time:164110ms step_avg:101.12ms
step:1624/1770 train_time:164214ms step_avg:101.12ms
step:1625/1770 train_time:164320ms step_avg:101.12ms
step:1625/1770 val_loss:3.3086 train_time:164423ms step_avg:101.18ms
step:1626/1770 train_time:164445ms step_avg:101.13ms
step:1627/1770 train_time:164533ms step_avg:101.13ms
step:1628/1770 train_time:164638ms step_avg:101.13ms
step:1629/1770 train_time:164742ms step_avg:101.13ms
step:1630/1770 train_time:164847ms step_avg:101.13ms
step:1631/1770 train_time:164951ms step_avg:101.14ms
step:1632/1770 train_time:165056ms step_avg:101.14ms
step:1633/1770 train_time:165161ms step_avg:101.14ms
step:1634/1770 train_time:165266ms step_avg:101.14ms
step:1635/1770 train_time:165371ms step_avg:101.14ms
step:1636/1770 train_time:165477ms step_avg:101.15ms
step:1637/1770 train_time:165584ms step_avg:101.15ms
step:1638/1770 train_time:165689ms step_avg:101.15ms
step:1639/1770 train_time:165795ms step_avg:101.16ms
step:1640/1770 train_time:165900ms step_avg:101.16ms
step:1641/1770 train_time:166006ms step_avg:101.16ms
step:1642/1770 train_time:166111ms step_avg:101.16ms
step:1643/1770 train_time:166216ms step_avg:101.17ms
step:1644/1770 train_time:166324ms step_avg:101.17ms
step:1645/1770 train_time:166429ms step_avg:101.17ms
step:1646/1770 train_time:166536ms step_avg:101.18ms
step:1647/1770 train_time:166642ms step_avg:101.18ms
step:1648/1770 train_time:166747ms step_avg:101.18ms
step:1649/1770 train_time:166853ms step_avg:101.18ms
step:1650/1770 train_time:166958ms step_avg:101.19ms
step:1651/1770 train_time:167063ms step_avg:101.19ms
step:1652/1770 train_time:167169ms step_avg:101.19ms
step:1653/1770 train_time:167275ms step_avg:101.19ms
step:1654/1770 train_time:167383ms step_avg:101.20ms
step:1655/1770 train_time:167491ms step_avg:101.20ms
step:1656/1770 train_time:167596ms step_avg:101.21ms
step:1657/1770 train_time:167702ms step_avg:101.21ms
step:1658/1770 train_time:167809ms step_avg:101.21ms
step:1659/1770 train_time:167916ms step_avg:101.22ms
step:1660/1770 train_time:168021ms step_avg:101.22ms
step:1661/1770 train_time:168126ms step_avg:101.22ms
step:1662/1770 train_time:168232ms step_avg:101.22ms
step:1663/1770 train_time:168336ms step_avg:101.22ms
step:1664/1770 train_time:168442ms step_avg:101.23ms
step:1665/1770 train_time:168547ms step_avg:101.23ms
step:1666/1770 train_time:168654ms step_avg:101.23ms
step:1667/1770 train_time:168759ms step_avg:101.24ms
step:1668/1770 train_time:168864ms step_avg:101.24ms
step:1669/1770 train_time:168969ms step_avg:101.24ms
step:1670/1770 train_time:169074ms step_avg:101.24ms
step:1671/1770 train_time:169180ms step_avg:101.24ms
step:1672/1770 train_time:169286ms step_avg:101.25ms
step:1673/1770 train_time:169394ms step_avg:101.25ms
step:1674/1770 train_time:169498ms step_avg:101.25ms
step:1675/1770 train_time:169603ms step_avg:101.26ms
step:1676/1770 train_time:169708ms step_avg:101.26ms
step:1677/1770 train_time:169817ms step_avg:101.26ms
step:1678/1770 train_time:169922ms step_avg:101.26ms
step:1679/1770 train_time:170028ms step_avg:101.27ms
step:1680/1770 train_time:170133ms step_avg:101.27ms
step:1681/1770 train_time:170239ms step_avg:101.27ms
step:1682/1770 train_time:170347ms step_avg:101.28ms
step:1683/1770 train_time:170453ms step_avg:101.28ms
step:1684/1770 train_time:170557ms step_avg:101.28ms
step:1685/1770 train_time:170661ms step_avg:101.28ms
step:1686/1770 train_time:170769ms step_avg:101.29ms
step:1687/1770 train_time:170876ms step_avg:101.29ms
step:1688/1770 train_time:170981ms step_avg:101.29ms
step:1689/1770 train_time:171087ms step_avg:101.29ms
step:1690/1770 train_time:171191ms step_avg:101.30ms
step:1691/1770 train_time:171297ms step_avg:101.30ms
step:1692/1770 train_time:171404ms step_avg:101.30ms
step:1693/1770 train_time:171511ms step_avg:101.31ms
step:1694/1770 train_time:171616ms step_avg:101.31ms
step:1695/1770 train_time:171721ms step_avg:101.31ms
step:1696/1770 train_time:171827ms step_avg:101.31ms
step:1697/1770 train_time:171935ms step_avg:101.32ms
step:1698/1770 train_time:172039ms step_avg:101.32ms
step:1699/1770 train_time:172144ms step_avg:101.32ms
step:1700/1770 train_time:172250ms step_avg:101.32ms
step:1701/1770 train_time:172355ms step_avg:101.33ms
step:1702/1770 train_time:172460ms step_avg:101.33ms
step:1703/1770 train_time:172565ms step_avg:101.33ms
step:1704/1770 train_time:172672ms step_avg:101.33ms
step:1705/1770 train_time:172777ms step_avg:101.34ms
step:1706/1770 train_time:172882ms step_avg:101.34ms
step:1707/1770 train_time:172990ms step_avg:101.34ms
step:1708/1770 train_time:173096ms step_avg:101.34ms
step:1709/1770 train_time:173202ms step_avg:101.35ms
step:1710/1770 train_time:173313ms step_avg:101.35ms
step:1711/1770 train_time:173421ms step_avg:101.36ms
step:1712/1770 train_time:173527ms step_avg:101.36ms
step:1713/1770 train_time:173632ms step_avg:101.36ms
step:1714/1770 train_time:173737ms step_avg:101.36ms
step:1715/1770 train_time:173843ms step_avg:101.37ms
step:1716/1770 train_time:173949ms step_avg:101.37ms
step:1717/1770 train_time:174055ms step_avg:101.37ms
step:1718/1770 train_time:174161ms step_avg:101.37ms
step:1719/1770 train_time:174271ms step_avg:101.38ms
step:1720/1770 train_time:174378ms step_avg:101.38ms
step:1721/1770 train_time:174484ms step_avg:101.39ms
step:1722/1770 train_time:174593ms step_avg:101.39ms
step:1723/1770 train_time:174701ms step_avg:101.39ms
step:1724/1770 train_time:174809ms step_avg:101.40ms
step:1725/1770 train_time:174918ms step_avg:101.40ms
step:1726/1770 train_time:175025ms step_avg:101.41ms
step:1727/1770 train_time:175132ms step_avg:101.41ms
step:1728/1770 train_time:175241ms step_avg:101.41ms
step:1729/1770 train_time:175347ms step_avg:101.42ms
step:1730/1770 train_time:175455ms step_avg:101.42ms
step:1731/1770 train_time:175562ms step_avg:101.42ms
step:1732/1770 train_time:175667ms step_avg:101.42ms
step:1733/1770 train_time:175776ms step_avg:101.43ms
step:1734/1770 train_time:175881ms step_avg:101.43ms
step:1735/1770 train_time:175988ms step_avg:101.43ms
step:1736/1770 train_time:176095ms step_avg:101.44ms
step:1737/1770 train_time:176200ms step_avg:101.44ms
step:1738/1770 train_time:176307ms step_avg:101.44ms
step:1739/1770 train_time:176413ms step_avg:101.44ms
step:1740/1770 train_time:176519ms step_avg:101.45ms
step:1741/1770 train_time:176626ms step_avg:101.45ms
step:1742/1770 train_time:176735ms step_avg:101.46ms
step:1743/1770 train_time:176841ms step_avg:101.46ms
step:1744/1770 train_time:176947ms step_avg:101.46ms
step:1745/1770 train_time:177053ms step_avg:101.46ms
step:1746/1770 train_time:177162ms step_avg:101.47ms
step:1747/1770 train_time:177267ms step_avg:101.47ms
step:1748/1770 train_time:177375ms step_avg:101.47ms
step:1749/1770 train_time:177481ms step_avg:101.48ms
step:1750/1770 train_time:177587ms step_avg:101.48ms
step:1750/1770 val_loss:3.2818 train_time:177692ms step_avg:101.54ms
step:1751/1770 train_time:177713ms step_avg:101.49ms
step:1752/1770 train_time:177805ms step_avg:101.49ms
step:1753/1770 train_time:177912ms step_avg:101.49ms
step:1754/1770 train_time:178018ms step_avg:101.49ms
step:1755/1770 train_time:178124ms step_avg:101.50ms
step:1756/1770 train_time:178231ms step_avg:101.50ms
step:1757/1770 train_time:178336ms step_avg:101.50ms
step:1758/1770 train_time:178442ms step_avg:101.50ms
step:1759/1770 train_time:178549ms step_avg:101.51ms
step:1760/1770 train_time:178656ms step_avg:101.51ms
step:1761/1770 train_time:178763ms step_avg:101.51ms
step:1762/1770 train_time:178872ms step_avg:101.52ms
step:1763/1770 train_time:178978ms step_avg:101.52ms
step:1764/1770 train_time:179085ms step_avg:101.52ms
step:1765/1770 train_time:179190ms step_avg:101.52ms
step:1766/1770 train_time:179300ms step_avg:101.53ms
step:1767/1770 train_time:179405ms step_avg:101.53ms
step:1768/1770 train_time:179512ms step_avg:101.53ms
step:1769/1770 train_time:179617ms step_avg:101.54ms
step:1770/1770 train_time:179723ms step_avg:101.54ms
step:1770/1770 val_loss:3.2788 train_time:179828ms step_avg:101.60ms
peak memory allocated: 30724 MiB reserved: 45952 MiB