From ced57a438218fb1be08f01f4a9b5bb3467013d4a Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 15 Jan 2019 13:08:25 +0200 Subject: [PATCH 1/2] state: don't forget to compute size of an individual file Fixes #1488 Signed-off-by: Ruslan Kuprieiev --- dvc/state.py | 5 ++++- tests/unit/state.py | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dvc/state.py b/dvc/state.py index 836b59ec3..2eb01750c 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -245,10 +245,10 @@ class State(object): @staticmethod def mtime_and_size(path): - size = 0 mtime = os.path.getmtime(path) if os.path.isdir(path): + size = 0 for root, dirs, files in os.walk(path): for dname in dirs: p = os.path.join(root, dname) @@ -263,6 +263,9 @@ class State(object): m = st.st_mtime if m > mtime: mtime = m + else: + size = os.path.getsize(path) + # State of files handled by dvc is stored in db as TEXT. # We cast results to string for later comparisons with stored values. return str(int(nanotime.timestamp(mtime))), str(size) diff --git a/tests/unit/state.py b/tests/unit/state.py index b1608df10..633424bcf 100644 --- a/tests/unit/state.py +++ b/tests/unit/state.py @@ -1,13 +1,21 @@ +import os + from dvc.state import State from tests.basic_env import TestDvc -class TestMtimeAndSizeReturnType(TestDvc): +class TestMtimeAndSize(TestDvc): def test(self): file_time, file_size = State.mtime_and_size(self.DATA) dir_time, dir_size = State.mtime_and_size(self.DATA_DIR) + actual_file_size = os.path.getsize(self.DATA) + actual_dir_size = (os.path.getsize(self.DATA) + + os.path.getsize(self.DATA_SUB)) + self.assertIs(type(file_time), str) self.assertIs(type(file_size), str) + self.assertEqual(file_size, str(actual_file_size)) self.assertIs(type(dir_time), str) self.assertIs(type(dir_size), str) + self.assertEqual(dir_size, str(actual_dir_size)) From 60e9ed4f0b7355965565ddf92a9278e7c0cf490c Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 15 Jan 2019 17:17:15 +0200 Subject: [PATCH 2/2] state: account for directories when computing size Signed-off-by: Ruslan Kuprieiev --- dvc/state.py | 14 +++----------- tests/unit/state.py | 4 +++- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/dvc/state.py b/dvc/state.py index 2eb01750c..655d9dfd9 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -245,26 +245,18 @@ class State(object): @staticmethod def mtime_and_size(path): + size = os.path.getsize(path) mtime = os.path.getmtime(path) if os.path.isdir(path): - size = 0 for root, dirs, files in os.walk(path): - for dname in dirs: - p = os.path.join(root, dname) - m = os.path.getmtime(p) - if m > mtime: - mtime = m - - for fname in files: - p = os.path.join(root, fname) + for name in dirs + files: + p = os.path.join(root, name) st = os.stat(p) size += st.st_size m = st.st_mtime if m > mtime: mtime = m - else: - size = os.path.getsize(path) # State of files handled by dvc is stored in db as TEXT. # We cast results to string for later comparisons with stored values. diff --git a/tests/unit/state.py b/tests/unit/state.py index 633424bcf..18d6060a6 100644 --- a/tests/unit/state.py +++ b/tests/unit/state.py @@ -10,7 +10,9 @@ class TestMtimeAndSize(TestDvc): dir_time, dir_size = State.mtime_and_size(self.DATA_DIR) actual_file_size = os.path.getsize(self.DATA) - actual_dir_size = (os.path.getsize(self.DATA) + actual_dir_size = (os.path.getsize(self.DATA_DIR) + + os.path.getsize(self.DATA) + + os.path.getsize(self.DATA_SUB_DIR) + os.path.getsize(self.DATA_SUB)) self.assertIs(type(file_time), str)