From cd877e09c574ee879d56a6980d04c13f22522139 Mon Sep 17 00:00:00 2001
From: Dmitry Petrov <dmitry.petrov@nevesomo.com>
Date: Fri, 21 Apr 2017 19:02:33 -0700
Subject: [PATCH] readme ...

---
 README.md | 116 +++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 101 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index a3b4b401b..ec88f2a36 100644
--- a/README.md
+++ b/README.md
@@ -30,12 +30,12 @@ Initialize dvc environment:
 
 ```
 $ dvc init
-Info. Directories data/, cache/ and state/ were created
-Info. File .gitignore was created
-Info. Directory cache was added to .gitignore file
-Info. [Git] A new commit a5ed2b9 was made in the current branch. Added files:
-Info. [Git] A  .gitignore
-Info. [Git] A  dvc.conf
+Directories data/, .cache/ and .state/ were created
+File .gitignore was created
+Directory .cache was added to .gitignore file
+[Git] A new commit e7a4b0d was made in the current branch. Added files:
+[Git]   A  .gitignore
+[Git]   A  dvc.conf
 ```
 
 The output shows the actions that dvc has made.
@@ -59,11 +59,10 @@ This parameter might be applied to any dvc command.
 Data file has to be imported in dvc `data/` directory before using it.
 
 ```
-$ dvc data-import https://s3-us-west-2.amazonaws.com/dataversioncontrol/functests/stackoverflow_raw_small/Badges.xml data/
-$ dvc data-import https://s3-us-west-2.amazonaws.com/dvc-doc/small/Badges.xml data/
-Info. [Git] A new commit 9f27943 was made in the current branch. Added files:
-Info. [Git] A  data/Badges.xml
-Info. [Git] A  state/Badges.xml.state
+$ dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/small/Posts.xml.tgz data/
+[Git] A new commit b69ab31 was made in the current branch. Added files:
+[Git]   A  .state/Posts.xml.tgz.state
+[Git]   A  data/Posts.xml.tgz
 ```
 
 The data file was downloaded and now it is available in the data directory.
@@ -74,8 +73,8 @@ However, it is only a part of the work that dvc has done.
 If you look carefully at the file you could see that this is actually a symlink to
     a cache directory file Badges.xml_a5ed2b9:
 ```
-$ ls -l data/Badges.xml
-lrwxr-xr-x  1 dmitry  staff    27B Apr 17 16:19 data/Badges.xml@ -> ../cache/Badges.xml_a5ed2b9
+$ ls -l data/Posts.xml.tgz
+lrwxr-xr-x  1 dmitry  staff    31B Apr 21 17:56 data/Posts.xml.tgz@ -> ../.cache/Posts.xml.tgz_9185f92
 ```
 
 Instead of importing the file to `data/Badges.xml` the system imports the file in the cache directory
@@ -88,13 +87,100 @@ The file in the cache directory `cache/Badges.xml_a5ed2b9` has the same name as
 
 In addition to the cache file dvc creates a state file in the `state/` directory for each data file:
 ```
-$ ls -l state/Badges.xml.state
--rw-r--r--  1 dmitry  staff   654B Apr 17 16:19 state/Badges.xml.state
+$ ls -l .state/Posts.xml.tgz.state
+-rw-r--r--  1 dmitry  staff   437B Apr 21 17:56 .state/Posts.xml.tgz.state
 ```
 
 The state file contains information for file reproducibility.
 
 
+## Structure data
+
+
+```python
+import sys
+import xml.etree.ElementTree
+
+def print_usage(msg):
+    if msg:
+        sys.stderr.write('{}\n'.format(msg))
+    sys.stderr.write('Usage:\n')
+    sys.stderr.write('\tpython posts_to_tsv.py INPUT OUTPUT\n')
+
+
+def process_posts(fd_in, fd_out):
+    num = 1
+    for line in fd_in:
+        try:
+            attr = xml.etree.ElementTree.fromstring(line).attrib
+            items = (
+                attr.get('Id', ''),
+                attr.get('PostTypeId', ''),
+                attr.get('CreationDate', ''),
+                attr.get('Score', ''),
+                attr.get('ViewCount', ''),
+                attr.get('OwnerUserId', ''),
+                attr.get('LastActivityDate', ''),
+                attr.get('AnswerCount', ''),
+                attr.get('CommentCount', ''),
+                attr.get('FavoriteCount', ''),
+                attr.get('Title', ''),
+                attr.get('Body', '')
+            )
+            fmt = ('{}\t' * len(items))[:-1] + '\n'
+
+            fd_out.write(fmt.format(*items))
+
+            num += 1
+        except Exception as ex:
+            sys.stderr.write('Error in line {}: {}'.format(num, ex))
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print_usage('Paramet error.')
+        sys.exit(1)
+
+    input = sys.argv[1]
+    if not os.path.exist(input):
+        print_usage('Input file {} does not exist'.format(input))
+        sys.exit(1)
+
+    with open(input) as fd_in:
+        with open(sys.argv[2], 'w') as fd_out:
+            process_posts(fd_in, fd_out)
+```
+
+
+```
+$ cd data
+$ dvc run tar zxf Posts.xml.tgz
+[Git] A new commit fbff960 was made in the current branch. Added files:
+[Git]   A  .state/Posts.xml.state
+[Git]   A  data/Posts.xml
+$ cd ..
+```
+
+
+```
+$ wget -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/small/posts_to_tsv.py
+```
+
+```
+$ git add code/posts_to_tsv.py
+$ git commit -m 'Post to tsv script'
+[master 4a581d9] Post to tsv script
+ 1 file changed, 51 insertions(+)
+  create mode 100644 code/posts_to_tsv.py
+```
+
+
+```
+$ dvc run python code/posts_to_tsv.py data/Posts.xml data/Posts.tsv
+[Git] A new commit dc54e5a was made in the current branch. Added files:
+[Git]   A  .state/Posts.tsv.state
+[Git]   A  data/Posts.tsv
+```
+
 # Copyright
 
 This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root).