python: how to output to console and file at the same time?

method 1:

logger = setup_logger("NINJA", output_dir, 0)
logger.info("{}".format('Hello World'))
logger.info(args)

---------------------

import logging
import os
import sys


def setup_logger(name, save_dir, distributed_rank): 

    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    # don't log results for the non-master process 

    if distributed_rank > 0: 

        return logger
    ch = logging.StreamHandler(stream=sys.stdout)
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s")
    ch.setFormatter(formatter)
    logger.propagate = False    logger.addHandler(ch)

    if save_dir:         

        fh = logging.FileHandler(os.path.join(save_dir, "log.txt"), mode='w')
        fh.setLevel(logging.DEBUG)
        fh.setFormatter(formatter)
        logger.addHandler(fh)

    return logger

method 2:

# log
if cfg.log_to_file:
    ReDirectSTD(cfg.stdout_file, 'stdout', False)
    ReDirectSTD(cfg.stderr_file, 'stderr', False)
# dump the configuration to log.
import pprint
print('-' * 60)
print('cfg.__dict__')
pprint.pprint(cfg.__dict__)
print('-' * 60)

---------------------

class ReDirectSTD(object):
    """
    overwrites the sys.stdout or sys.stderr
    Args:
      fpath: file path
      console: one of ['stdout', 'stderr']
      immediately_visiable: False
    Usage example:
      ReDirectSTD('stdout.txt', 'stdout', False)
      ReDirectSTD('stderr.txt', 'stderr', False)
    """
    def __init__(self, fpath=None, console='stdout', immediately_visiable=False):
        import sys
        import os
        assert console in ['stdout', 'stderr']
        self.console = sys.stdout if console == "stdout" else sys.stderr
        self.file = fpath
        self.f = None
        self.immediately_visiable = immediately_visiable
        if fpath is not None:
            # Remove existing log file
            if os.path.exists(fpath):
                os.remove(fpath)
        if console == 'stdout':
            sys.stdout = self
        else:
            sys.stderr = self

    def __del__(self):
        self.close()

    def __enter__(self):
        pass

    def __exit__(self, **args):
        self.close()

    def write(self, msg):
        self.console.write(msg)
        if self.file is not None:
            if not os.path.exists(os.path.dirname(os.path.abspath(self.file))):
                os.mkdir(os.path.dirname(os.path.abspath(self.file)))
            if self.immediately_visiable:
                with open(self.file, 'a') as f:
                    f.write(msg)
            else:
                if self.f is None:
                    self.f = open(self.file, 'w')
                self.f.write(msg)

    def flush(self):
        self.console.flush()
        if self.f is not None:
            self.f.flush()
            import os
            os.fsync(self.f.fileno())
   
    def close(self):
        self.console.close()
        if self.f is not None:
            self.f.close()

string, numpy, convert, "./output/[array(['hello-world'], dtype=U22)]/12454.png"

string, numpy, convert, "./output/[array(['hello-world'], dtype='<U22')]/12454.png" 

original is

att_list[idx] 

correct one should be

att_list[idx][0][0]

how to refresh nvidia-docker2

sudo systemctl daemon-reload
sudo systemctl restart docker

docker: Error response from daemon: Unknown runtime specified nvidia.

$ sudo gedit /etc/docker/daemon.json
{
    "runtimes": {
        "nvidia": {
            "path": "nvidia-container-runtime",
            "runtimeArgs": []
        }
    }
}

how to test nvidia-docker

docker run --runtime=nvidia --rm nvidia/cuda nvidia-smi

docker run --rm --runtime=nvidia nvidia/cuda:10.0-devel nvidia-smi 

docker run --rm --runtime=nvidia vistart/cuda:10.0-cudnn7-devel-ubuntu16.04 nvidia-smi 

How to install nvidia driver into kernel?

ctrl + alt + F2

login: username

password: password

sudo apt purge nvidia* 

sudo service lightdm stop 

sudo sh ./NVIDIA-Linux-x86_64-410.104.run

sudo service lightdm start 

/usr/bin/ld: cannot find -lboost_python3

cd /usr/lib/x86_64-linux-gnu

sudo ln -s libboost_python-py35.so libboost_python3.so 

/home/user/anaconda3/lib/libtbb.so.2: undefined reference to `__cxa_init_primary_exception@CXXABI_1.3.11'

conda uninstall tbb

../../lib/libopencv_imgcodecs.so.3.4.4: undefined reference to `TIFFReadRGBAStrip@LIBTIFF_4.0'

sudo apt remove libopencv*

conda uninstall libtiff

sudo su
make all -j16

how to get the current folder / directory size

to get the harddisk size
df -h .

to get the current directory size
du -hc .

to get the home folder size
du -sh /home

to get the current sub directory sorted size
du -h --max-depth=1 | sort -hr

how to export list into a file and import it again? use pickle

You can use pickle module for that. This module have two methods,

1. Pickling(dump): Convert Python objects into string representation.
2. Unpickling(load): Retrieving original objects from stored string representstion. 

import pickle
l = [1,2,3,4]
with open("test.txt", "wb") as fp: #Pickling
    pickle.dump(l, fp)
 

with open("test.txt", "rb") as fp: # Unpickling
    b = pickle.load(fp)
b
 
[1, 2, 3, 4]

How to tar with multi thread / core

You can use pigz instead of gzip, which does gzip compression on multiple cores. Instead of using the -z option, you would pipe it through pigz:

tar cf - paths-to-archive | pigz > archive.tar.gz



By default, pigz uses the number of available cores, or eight if it could not query that. You can ask for more with -p n, e.g. -p 32. pigz has the same options as gzip, so you can request better compression with -9. E.g.

tar cf - paths-to-archive | pigz -9 -p 32 > archive.tar.gz

how to overwrite the sys.stdout or sys.stderr

class ReDirectSTD(object):
    """
    overwrites the sys.stdout or sys.stderr
    Args:
      fpath: file path
      console: one of ['stdout', 'stderr']
      immediately_visiable: False
    Usage example:
      ReDirectSTD('stdout.txt', 'stdout', False)
      ReDirectSTD('stderr.txt', 'stderr', False)
    """
    def __init__(self, fpath=None, console='stdout', immediately_visiable=False):
        import sys
        import os
        assert console in ['stdout', 'stderr']
        self.console = sys.stdout if console == "stdout" else sys.stderr
        self.file = fpath
        self.f = None
        self.immediately_visiable = immediately_visiable
        if fpath is not None:
            # Remove existing log file
            if os.path.exists(fpath):
                os.remove(fpath)
        if console == 'stdout':
            sys.stdout = self
        else:
            sys.stderr = self

    def __del__(self):
        self.close()

    def __enter__(self):
        pass

    def __exit__(self, **args):
        self.close()

    def write(self, msg):
        self.console.write(msg)
        if self.file is not None:
            if not os.path.exists(os.path.dirname(os.path.abspath(self.file))):
                os.mkdir(os.path.dirname(os.path.abspath(self.file)))
            if self.immediately_visiable:
                with open(self.file, 'a') as f:
                    f.write(msg)
            else:
                if self.f is None:
                    self.f = open(self.file, 'w')
                self.f.write(msg)

    def flush(self):
        self.console.flush()
        if self.f is not None:
            self.f.flush()
            import os
            os.fsync(self.f.fileno())
   
    def close(self):
        self.console.close()
        if self.f is not None:
            self.f.close()