Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1647 from pared/1526

Browse Source 
remote local/state : add check for cache dir metadata changes
This commit is contained in:
Ruslan Kuprieiev
2019-02-25 20:11:44 +02:00
committed by GitHub
parent 38536903e1 4464e29c15
commit 68603f012e
6 changed files with 135 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/remote/local.py
+19 -7
View File
@@ -21,6 +21,8 @@ from dvc.exceptions import DvcException
from dvc.progress import progress
from concurrent.futures import ThreadPoolExecutor
from dvc.utils.fs import get_mtime_and_size, get_inode
class RemoteLOCAL(RemoteBase):
scheme = "local"
@@ -122,13 +124,11 @@ class RemoteLOCAL(RemoteBase):
if self.changed_cache_file(md5):
return True
if not self.is_dir_cache(cache):
continue
for entry in self.load_dir_cache(md5):
md5 = entry[self.PARAM_CHECKSUM]
cache = self.get(md5)
clist.append((cache, md5))
if self.is_dir_cache(cache) and self._cache_metadata_changed():
for entry in self.load_dir_cache(md5):
md5 = entry[self.PARAM_CHECKSUM]
cache = self.get(md5)
clist.append((cache, md5))
return False
@@ -695,3 +695,15 @@ class RemoteLOCAL(RemoteBase):
show_checksums=show_checksums,
download=True,
)
def _cache_metadata_changed(self):
mtime, size = get_mtime_and_size(self.cache_dir)
inode = get_inode(self.cache_dir)
existing_record = self.state.get_state_record_for_inode(inode)
if existing_record:
cached_mtime, cached_size, _, _ = existing_record
return not (mtime == cached_mtime and size == cached_size)
return True
dvc/state.py
+48 -73
View File
@@ -2,18 +2,14 @@
from __future__ import unicode_literals
from dvc.utils.compat import str
import os
import time
import sqlite3
import nanotime
import dvc.logger as logger
from dvc.config import Config
from dvc.system import System
from dvc.utils import file_md5, remove
from dvc.utils import file_md5, remove, current_timestamp
from dvc.exceptions import DvcException
from dvc.utils.fs import get_mtime_and_size, get_inode
class StateVersionTooNewError(DvcException):
@@ -289,6 +285,8 @@ class State(object): # pylint: disable=too-many-instance-attributes
)
)
self._update_cache_directory_state()
self.database.commit()
self.cursor.close()
self.database.close()
@@ -296,30 +294,6 @@ class State(object): # pylint: disable=too-many-instance-attributes
self.cursor = None
self.inserts = 0
@staticmethod
def _mtime_and_size(path):
size = os.path.getsize(path)
mtime = os.path.getmtime(path)
if os.path.isdir(path):
for root, dirs, files in os.walk(str(path)):
for name in dirs + files:
entry = os.path.join(root, name)
stat = os.stat(entry)
size += stat.st_size
entry_mtime = stat.st_mtime
if entry_mtime > mtime:
mtime = entry_mtime
# State of files handled by dvc is stored in db as TEXT.
# We cast results to string for later comparisons with stored values.
return str(int(nanotime.timestamp(mtime))), str(size)
@staticmethod
def _inode(path):
logger.debug("Path {} inode {}".format(path, System.inode(path)))
return System.inode(path)
def _do_update(self, path, known_checksum=None):
"""
Make sure the stored info for the given path is up to date.
@@ -327,25 +301,24 @@ class State(object): # pylint: disable=too-many-instance-attributes
if not os.path.exists(path):
return None, None
actual_mtime, actual_size = self._mtime_and_size(path)
actual_inode = self._inode(path)
actual_mtime, actual_size = get_mtime_and_size(path)
actual_inode = get_inode(path)
existing_records = self._get_state_records_for_inode(actual_inode)
should_insert_new_record = not existing_records
existing_record = self.get_state_record_for_inode(actual_inode)
if should_insert_new_record:
md5, info = self._insert_new_state_record(
path, actual_inode, actual_mtime, actual_size, known_checksum
)
else:
if existing_record:
md5, info = self._update_existing_state_record(
path,
actual_inode,
actual_mtime,
actual_size,
existing_records,
existing_record,
known_checksum,
)
else:
md5, info = self._insert_new_state_record(
path, actual_inode, actual_mtime, actual_size, known_checksum
)
return md5, info
@@ -355,13 +328,11 @@ class State(object): # pylint: disable=too-many-instance-attributes
actual_inode,
actual_mtime,
actual_size,
existing_records,
existing_record,
known_checksum=None,
):
md5, mtime, size = self._get_existing_record_data(
actual_inode, actual_mtime, actual_size, existing_records
)
mtime, size, md5, _ = existing_record
if _file_metadata_changed(actual_mtime, mtime, actual_size, size):
md5, info = self._update_state_for_path_changed(
path, actual_inode, actual_mtime, actual_size, known_checksum
@@ -371,28 +342,12 @@ class State(object): # pylint: disable=too-many-instance-attributes
self._update_state_record_timestamp_for_inode(actual_inode)
return md5, info
def _get_existing_record_data(
self, actual_inode, actual_mtime, actual_size, existing_records
):
assert len(existing_records) == 1
assert len(existing_records[0]) == 5
inode, mtime, size, md5, _ = existing_records[0]
inode = self._from_sqlite(inode)
assert inode == actual_inode
logger.debug(
"Inode '{}', mtime '{}', actual mtime '{}', size '{}', "
"actual size '{}'.".format(
inode, mtime, actual_mtime, size, actual_size
)
)
return md5, mtime, size
def _update_state_record_timestamp_for_inode(self, actual_inode):
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
self._execute(
cmd.format(
self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
current_timestamp(),
self._to_sqlite(actual_inode),
)
)
@@ -421,7 +376,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
actual_mtime,
actual_size,
md5,
int(nanotime.timestamp(time.time())),
current_timestamp(),
self._to_sqlite(actual_inode),
)
)
@@ -445,19 +400,22 @@ class State(object): # pylint: disable=too-many-instance-attributes
actual_mtime,
actual_size,
md5,
int(nanotime.timestamp(time.time())),
current_timestamp(),
)
)
self.inserts += 1
return md5, info
def _get_state_records_for_inode(self, actual_inode):
cmd = "SELECT * from {} WHERE inode={}".format(
self.STATE_TABLE, self._to_sqlite(actual_inode)
)
def get_state_record_for_inode(self, inode):
cmd = "SELECT mtime, size, md5, timestamp from {} " "WHERE inode={}"
cmd = cmd.format(self.STATE_TABLE, inode)
self._execute(cmd)
ret = self._fetchall()
return ret
results = self._fetchall()
if results:
# uniquness constrain on inode
assert len(results) == 1
return results[0]
return None
def update(self, path, known_checksum=None):
"""Gets the checksum for the specified path. Checksum will be
@@ -499,8 +457,8 @@ class State(object): # pylint: disable=too-many-instance-attributes
if not os.path.exists(path):
return
mtime, _ = self._mtime_and_size(path)
inode = self._inode(path)
mtime, _ = get_mtime_and_size(path)
inode = get_inode(path)
relpath = os.path.relpath(path, self.root_dir)
cmd = (
@@ -531,8 +489,8 @@ class State(object): # pylint: disable=too-many-instance-attributes
if not os.path.exists(path):
continue
actual_inode = self._inode(path)
actual_mtime, _ = self._mtime_and_size(path)
actual_inode = get_inode(path)
actual_mtime, _ = get_mtime_and_size(path)
if inode == actual_inode and mtime == actual_mtime:
logger.debug("Removing '{}' as unused link.".format(path))
@@ -542,3 +500,20 @@ class State(object): # pylint: disable=too-many-instance-attributes
for relpath in unused:
cmd = 'DELETE FROM {} WHERE path = "{}"'
self._execute(cmd.format(self.LINK_STATE_TABLE, relpath))
def _update_cache_directory_state(self):
cache_path = self.repo.cache.local.cache_dir
mtime, size = get_mtime_and_size(cache_path)
inode = get_inode(cache_path)
cmd = (
"INSERT OR REPLACE INTO {}(inode, size, mtime, timestamp, md5) "
'VALUES ({}, "{}", "{}", "{}", "")'.format(
self.STATE_TABLE,
self._to_sqlite(inode),
size,
mtime,
current_timestamp(),
)
)
self._execute(cmd)
dvc/utils/__init__.py
+6
View File
@@ -10,6 +10,8 @@ import math
import json
import shutil
import hashlib
import nanotime
import time
LOCAL_CHUNK_SIZE = 1024 * 1024
LARGE_FILE_SIZE = 1024 * 1024 * 1024
@@ -219,3 +221,7 @@ def tmp_fname(fname):
from uuid import uuid4
return fname + "." + str(uuid4()) + ".tmp"
def current_timestamp():
return int(nanotime.timestamp(time.time()))
dvc/utils/fs.py
+33
View File
@@ -0,0 +1,33 @@
from __future__ import unicode_literals
import nanotime
import os
import dvc.logger as logger
from dvc.system import System
from dvc.utils.compat import str
def get_inode(path):
inode = System.inode(path)
logger.debug("Path {} inode {}".format(path, inode))
return inode
def get_mtime_and_size(path):
size = os.path.getsize(path)
mtime = os.path.getmtime(path)
if os.path.isdir(path):
for root, dirs, files in os.walk(str(path)):
for name in dirs + files:
entry = os.path.join(root, name)
stat = os.stat(entry)
size += stat.st_size
entry_mtime = stat.st_mtime
if entry_mtime > mtime:
mtime = entry_mtime
# State of files handled by dvc is stored in db as TEXT.
# We cast results to string for later comparisons with stored values.
return str(int(nanotime.timestamp(mtime))), str(size)
tests/test_add.py
+23
View File
@@ -276,3 +276,26 @@ class TestAddCommit(TestDvc):
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.FOO))
self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1)
class TestShouldNotCheckCacheForDirIfCacheMetadataDidNotChange(TestDvc):
def test(self):
remote_local_loader_spy = spy(
dvc.remote.local.RemoteLOCAL.load_dir_cache
)
with patch.object(
dvc.remote.local.RemoteLOCAL,
"load_dir_cache",
remote_local_loader_spy,
):
ret = main(["config", "cache.type", "copy"])
self.assertEqual(ret, 0)
ret = main(["add", self.DATA_DIR])
self.assertEqual(ret, 0)
self.assertEqual(1, remote_local_loader_spy.mock.call_count)
ret = main(["status", "{}.dvc".format(self.DATA_DIR)])
self.assertEqual(ret, 0)
self.assertEqual(1, remote_local_loader_spy.mock.call_count)
tests/unit/state.py → tests/unit/fs.py
+6 -7
View File
@@ -1,15 +1,14 @@
from dvc.utils.compat import str
import os
from dvc.state import State
from tests.basic_env import TestDvc
from dvc.utils.compat import str
from dvc.utils.fs import get_mtime_and_size
from tests.basic_env import TestDir
class TestMtimeAndSize(TestDvc):
class TestMtimeAndSize(TestDir):
def test(self):
file_time, file_size = State._mtime_and_size(self.DATA)
dir_time, dir_size = State._mtime_and_size(self.DATA_DIR)
file_time, file_size = get_mtime_and_size(self.DATA)
dir_time, dir_size = get_mtime_and_size(self.DATA_DIR)
actual_file_size = os.path.getsize(self.DATA)
actual_dir_size = (