xref: /titanic_41/usr/src/tools/onbld/Scm/Backup.py (revision aff4bce51ecc47df7e5a6351b7cee6bc20408c63)
1#
2#  This program is free software; you can redistribute it and/or modify
3#  it under the terms of the GNU General Public License version 2
4#  as published by the Free Software Foundation.
5#
6#  This program is distributed in the hope that it will be useful,
7#  but WITHOUT ANY WARRANTY; without even the implied warranty of
8#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9#  GNU General Public License for more details.
10#
11#  You should have received a copy of the GNU General Public License
12#  along with this program; if not, write to the Free Software
13#  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
14#
15
16#
17# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
18# Use is subject to license terms.
19#
20
21'''
22Workspace backup
23
24Backup format is:
25   backupdir/
26      wsname/
27         generation#/
28            dirstate (handled by CdmUncommittedBackup)
29                File containing dirstate nodeid (the changeset we need
30                to update the workspace to after applying the bundle).
31                This is the node to which the working copy changes
32                (see 'diff', below) will be applied if applicable.
33
34            bundle (handled by CdmCommittedBackup)
35                An Hg bundle containing outgoing committed changes.
36
37            nodes (handled by CdmCommittedBackup)
38                A text file listing the full (hex) nodeid of all nodes in
39                bundle, used by need_backup.
40
41            diff (handled by CdmUncommittedBackup)
42                A Git-formatted diff containing uncommitted changes.
43
44            renames (handled by CdmUncommittedBackup)
45                A list of renames in the working copy that have to be
46                applied manually, rather than by the diff.
47
48            metadata.tar.gz (handled by CdmMetadataBackup)
49                $CODEMGR_WS/.hg/hgrc
50                $CODEMGR_WS/.hg/localtags
51                $CODEMGR_WS/.hg/patches (Mq data)
52
53         latest -> generation#
54            Newest backup generation.
55
56All files in a given backup generation, with the exception of
57dirstate, are optional.
58'''
59
60import os, pwd, shutil, traceback, tarfile, time
61from mercurial import changegroup, patch, node, util, revlog
62
63
64class CdmNodeMissing(util.Abort):
65    '''a required node is not present in the destination workspace.
66
67    This may occur both in the case where the bundle contains a
68    changeset which is a child of a node not present in the
69    destination workspace (because the destination workspace is not as
70    up-to-date as the source), or because the source and destination
71    workspace are not related.
72
73    It may also happen in cases where the uncommitted changes need to
74    be applied onto a node that the workspace does not possess even
75    after application of the bundle (on a branch not present
76    in the bundle or destination workspace, for instance)'''
77
78    def __init__(self, msg, name):
79        #
80        # If e.name is a string 20 characters long, it is
81        # assumed to be a node.  (Mercurial makes this
82        # same assumption, when creating a LookupError)
83        #
84        if isinstance(name, str) and len(name) == 20:
85            n = node.short(name)
86        else:
87            n = name
88
89        util.Abort.__init__(self, "%s: changeset '%s' is missing\n"
90                            "Your workspace is either not "
91                            "sufficiently up to date,\n"
92                            "or is unrelated to the workspace from "
93                            "which the backup was taken.\n" % (msg, n))
94
95
96class CdmCommittedBackup(object):
97    '''Backup of committed changes'''
98
99    def __init__(self, backup, ws):
100        self.ws = ws
101        self.bu = backup
102        self.files = ('bundle', 'nodes')
103
104    def _outgoing_nodes(self, parent):
105        '''Return a list of all outgoing nodes in hex format'''
106
107        if parent:
108            outgoing = self.ws.findoutgoing(parent)
109            nodes = self.ws.repo.changelog.nodesbetween(outgoing)[0]
110            return map(node.hex, nodes)
111        else:
112            return []
113
114    def backup(self):
115        '''Backup committed changes'''
116        parent = self.ws.parent()
117
118        if not parent:
119            self.ws.ui.warn('Workspace has no parent, committed changes will '
120                            'not be backed up\n')
121            return
122
123        out = self.ws.findoutgoing(parent)
124        if not out:
125            return
126
127        cg = self.ws.repo.changegroup(out, 'bundle')
128        changegroup.writebundle(cg, self.bu.backupfile('bundle'), 'HG10BZ')
129
130        outnodes = self._outgoing_nodes(parent)
131        if outnodes:
132            fp = None
133            try:
134                try:
135                    fp = open(self.bu.backupfile('nodes'), 'w')
136                    fp.write('%s\n' % '\n'.join(outnodes))
137                except EnvironmentError, e:
138                    raise util.Abort("couldn't store outgoing nodes: %s" % e)
139            finally:
140                if fp and not fp.closed:
141                    fp.close()
142
143    def restore(self):
144        '''Restore committed changes from backup'''
145        bfile = self.bu.backupfile('bundle')
146
147        if os.path.exists(bfile):
148            f = None
149            try:
150                try:
151                    f = open(bfile, 'r')
152                    bundle = changegroup.readbundle(f, bfile)
153                    self.ws.repo.addchangegroup(bundle, 'strip',
154                                                'bundle:%s' % bfile)
155                except EnvironmentError, e:
156                    raise util.Abort("couldn't restore committed changes: %s\n"
157                                     "   %s" % (bfile, e))
158                except revlog.LookupError, e:
159                    raise CdmNodeMissing("couldn't restore committed changes",
160                                                     e.name)
161            finally:
162                if f and not f.closed:
163                    f.close()
164
165    def need_backup(self):
166        '''Compare backup of committed changes to workspace'''
167
168        if os.path.exists(self.bu.backupfile('nodes')):
169            f = None
170            try:
171                try:
172                    f = open(self.bu.backupfile('nodes'))
173                    bnodes = set([line.rstrip('\r\n')
174                                  for line in f.readlines()])
175                    f.close()
176                except EnvironmentError, e:
177                    raise util.Abort("couldn't open backup node list: %s" % e)
178            finally:
179                if f and not f.closed:
180                    f.close()
181        else:
182            bnodes = set()
183
184        outnodes = set(self._outgoing_nodes(self.ws.parent()))
185        if outnodes != bnodes:
186            return True
187
188        return False
189
190    def cleanup(self):
191        '''Remove backed up committed changes'''
192
193        for fname in self.files:
194            if os.path.exists(self.bu.backupfile(fname)):
195                os.unlink(self.bu.backupfile(fname))
196
197
198class CdmUncommittedBackup(object):
199    '''Backup of uncommitted changes'''
200
201    def __init__(self, backup, ws):
202        self.ws = ws
203        self.bu = backup
204
205    def _clobbering_renames(self):
206        '''Return a list of pairs of files representing renames/copies
207        that clobber already versioned files.  [(oldname newname)...]'''
208
209        #
210        # Note that this doesn't handle uncommitted merges
211        # as CdmUncommittedBackup itself doesn't.
212        #
213        wctx = self.ws.workingctx()
214        parent = wctx.parents()[0]
215
216        ret = []
217        for fname in wctx.added() + wctx.modified():
218            rn = wctx.filectx(fname).renamed()
219            if rn and fname in parent:
220                ret.append((rn[0], fname))
221        return ret
222
223    def backup(self):
224        '''Backup uncommitted changes'''
225
226        if self.ws.merged():
227            raise util.Abort("Unable to backup an uncommitted merge.\n"
228                             "Please complete your merge and commit")
229
230        dirstate = node.hex(self.ws.workingctx().parents()[0].node())
231
232        fp = None
233        try:
234            try:
235                fp = open(self.bu.backupfile('dirstate'), 'w')
236                fp.write(dirstate + '\n')
237            except EnvironmentError, e:
238                raise util.Abort("couldn't save working copy parent: %s" % e)
239        finally:
240            if fp and not fp.closed:
241                fp.close()
242
243        try:
244            try:
245                fp = open(self.bu.backupfile('renames'), 'w')
246                for cons in self._clobbering_renames():
247                    fp.write("%s %s\n" % cons)
248            except EnvironmentError, e:
249                raise util.Abort("couldn't save clobbering copies: %s" % e)
250        finally:
251            if fp and not fp.closed:
252                fp.close()
253
254        try:
255            try:
256                fp = open(self.bu.backupfile('diff'), 'w')
257                opts = patch.diffopts(self.ws.ui, opts={'git': True})
258                fp.write(self.ws.diff(opts=opts))
259            except EnvironmentError, e:
260                raise util.Abort("couldn't save working copy diff: %s" % e)
261        finally:
262            if fp and not fp.closed:
263                fp.close()
264
265    def _dirstate(self):
266        '''Return the desired working copy node from the backup'''
267        fp = None
268        try:
269            try:
270                fp = open(self.bu.backupfile('dirstate'))
271                dirstate = fp.readline().strip()
272                return dirstate
273            except EnvironmentError, e:
274                raise util.Abort("couldn't read saved parent: %s" % e)
275        finally:
276            if fp and not fp.closed:
277                fp.close()
278
279    def restore(self):
280        '''Restore uncommitted changes'''
281        diff = self.bu.backupfile('diff')
282        dirstate = self._dirstate()
283
284        #
285        # Check that the patch's parent changeset exists.
286        #
287        try:
288            n = node.bin(dirstate)
289            self.ws.repo.changelog.lookup(n)
290        except revlog.LookupError, e:
291            raise CdmNodeMissing("couldn't restore uncommitted changes",
292                                 e.name)
293
294        try:
295            self.ws.clean(rev=dirstate)
296        except util.Abort, e:
297            raise util.Abort("couldn't update to saved node: %s" % e)
298
299        if not os.path.exists(diff):
300            return
301
302        #
303        # There's a race here whereby if the patch (or part thereof)
304        # is applied within the same second as the clean above (such
305        # that mtime doesn't change) and if the size of that file
306        # does not change, Hg may not see the change.
307        #
308        # We sleep a full second to avoid this, as sleeping merely
309        # until the next second begins would require very close clock
310        # synchronization on network filesystems.
311        #
312        time.sleep(1)
313
314        files = {}
315        try:
316            try:
317                fuzz = patch.patch(diff, self.ws.ui, strip=1,
318                                   cwd=self.ws.repo.root, files=files)
319                if fuzz:
320                    raise util.Abort('working copy diff applied with fuzz')
321            except Exception, e:
322                raise util.Abort("couldn't apply working copy diff: %s\n"
323                                 "   %s" % (diff, e))
324        finally:
325            patch.updatedir(self.ws.ui, self.ws.repo, files)
326
327        if not os.path.exists(self.bu.backupfile('renames')):
328            return
329
330        #
331        # We need to re-apply name changes where the new name
332        # (rename/copy destination) is an already versioned file, as
333        # Hg would otherwise ignore them.
334        #
335        try:
336            fp = open(self.bu.backupfile('renames'))
337            for line in fp:
338                source, dest = line.strip().split()
339                self.ws.repo.copy(source, dest)
340        except EnvironmentError, e:
341            raise util.Abort('unable to open renames file: %s' % e)
342        except ValueError:
343            raise util.Abort('corrupt renames file: %s' %
344                             self.bu.backupfile('renames'))
345
346    def need_backup(self):
347        '''Compare backup of uncommitted changes to workspace'''
348        cnode = self.ws.workingctx().parents()[0].node()
349        if self._dirstate() != node.hex(cnode):
350            return True
351
352        opts = patch.diffopts(self.ws.ui, opts={'git': True})
353        curdiff = self.ws.diff(opts=opts)
354
355        diff = self.bu.backupfile('diff')
356        if os.path.exists(diff):
357            try:
358                try:
359                    fd = open(diff)
360                    backdiff = fd.read()
361                except EnvironmentError, e:
362                    raise util.Abort("couldn't open backup diff %s\n"
363                                     "   %s" % (diff, e))
364            finally:
365                if fd and not fd.closed:
366                    fd.close()
367        else:
368            backdiff = ''
369
370        if backdiff != curdiff:
371            return True
372
373
374        currrenamed = self._clobbering_renames()
375        bakrenamed = None
376
377        if os.path.exists(self.bu.backupfile('renames')):
378            try:
379                try:
380                    fd = open(self.bu.backupfile('renames'))
381                    bakrenamed = [line.strip().split(' ') for line in fd]
382                except EnvironmentError, e:
383                    raise util.Abort("couldn't open renames file %s: %s\n" %
384                                     (self.bu.backupfile('renames'), e))
385            finally:
386                if fd and not fd.closed:
387                    fd.close()
388
389            if currrenamed != bakrenamed:
390                return True
391
392        return False
393
394    def cleanup(self):
395        '''Remove backed up uncommitted changes'''
396        for fname in ('dirstate', 'diff', 'renames'):
397            if os.path.exists(self.bu.backupfile(fname)):
398                os.unlink(self.bu.backupfile(fname))
399
400
401class CdmMetadataBackup(object):
402    '''Backup of workspace metadata'''
403
404    def __init__(self, backup, ws):
405        self.bu = backup
406        self.ws = ws
407        self.files = ('hgrc', 'localtags', 'patches', 'cdm')
408
409    def backup(self):
410        '''Backup workspace metadata'''
411
412        tar = None
413
414        try:
415            try:
416                tar = tarfile.open(self.bu.backupfile('metadata.tar.gz'),
417                                   'w:gz')
418                tar.errorlevel = 2
419            except (EnvironmentError, tarfile.TarError), e:
420                raise util.Abort("couldn't open %s for writing: %s" %
421                                 (self.bu.backupfile('metadata.tar.gz'), e))
422
423            try:
424                for elt in self.files:
425                    fpath = self.ws.repo.join(elt)
426                    if os.path.exists(fpath):
427                        tar.add(fpath, elt)
428            except (EnvironmentError, tarfile.TarError), e:
429                #
430                # tarfile.TarError doesn't include the tar member or file
431                # in question, so we have to do so ourselves.
432                #
433                if isinstance(e, tarfile.TarError):
434                    error = "%s: %s" % (elt, e)
435                else:
436                    error = str(e)
437
438                raise util.Abort("couldn't backup metadata to %s:\n"
439                                 "  %s" %
440                                 (self.bu.backupfile('metadata.tar.gz'),
441                                  error))
442        finally:
443            if tar and not tar.closed:
444                tar.close()
445
446    def old_restore(self):
447        '''Restore workspace metadata from an pre-tar backup'''
448
449        for fname in self.files:
450            bfile = self.bu.backupfile(fname)
451            wfile = self.ws.repo.join(fname)
452
453            if os.path.exists(bfile):
454                try:
455                    shutil.copy2(bfile, wfile)
456                except EnvironmentError, e:
457                    raise util.Abort("couldn't restore metadata from %s:\n"
458                                     "   %s" % (bfile, e))
459
460    def tar_restore(self):
461        '''Restore workspace metadata (from a tar-style backup)'''
462
463        if os.path.exists(self.bu.backupfile('metadata.tar.gz')):
464            tar = None
465
466            try:
467                try:
468                    tar = tarfile.open(self.bu.backupfile('metadata.tar.gz'))
469                    tar.errorlevel = 2
470                except (EnvironmentError, tarfile.TarError), e:
471                    raise util.Abort("couldn't open %s: %s" %
472                                 (self.bu.backupfile('metadata.tar.gz'), e))
473
474                try:
475                    for elt in tar:
476                        tar.extract(elt, path=self.ws.repo.path)
477                except (EnvironmentError, tarfile.TarError), e:
478                    # Make sure the member name is in the exception message.
479                    if isinstance(e, tarfile.TarError):
480                        error = "%s: %s" % (elt.name, e)
481                    else:
482                        error = str(e)
483
484                    raise util.Abort("couldn't restore metadata from %s:\n"
485                                     "   %s" %
486                                     (self.bu.backupfile('metadata.tar.gz'),
487                                      error))
488            finally:
489                if tar and not tar.closed:
490                    tar.close()
491
492    def restore(self):
493        '''Restore workspace metadata'''
494
495        if os.path.exists(self.bu.backupfile('hgrc')):
496            self.old_restore()
497        else:
498            self.tar_restore()
499
500    def need_backup(self):
501        '''Compare backed up workspace metadata to workspace'''
502
503        if os.path.exists(self.bu.backupfile('metadata.tar.gz')):
504            try:
505                tar = tarfile.open(self.bu.backupfile('metadata.tar.gz'))
506                tar.errorlevel = 2
507            except (EnvironmentError, tarfile.TarError), e:
508                raise util.Abort("couldn't open metadata tarball: %s\n"
509                                 "   %s" %
510                                 (self.bu.backupfile('metadata.tar.gz'), e))
511
512            for elt in tar:
513                fpath = self.ws.repo.join(elt.name)
514                if not os.path.exists(fpath):
515                    return True     # File in tar, not workspace
516
517                if elt.isdir():     # Don't care about directories
518                    continue
519
520                #
521                # The filesystem can give us mtime with fractional seconds
522                # (as a float), whereas tar files only keep it to the second.
523                #
524                # Always compare to the integer (second-granularity) mtime.
525                #
526                if (elt.mtime != int(os.path.getmtime(fpath)) or
527                    elt.size != os.path.getsize(fpath)):
528                    return True
529
530            tarnames = tar.getnames()
531            tar.close()
532        else:
533            tarnames = []
534
535        for mfile in self.files:
536            fpath = self.ws.repo.join(mfile)
537
538            if os.path.isdir(fpath):
539                # Directories in tarfile always end with a '/'
540                if not mfile.endswith('/'):
541                    mfile += '/'
542
543                if mfile not in tarnames:
544                    return True
545
546                for root, dirs, files in os.walk(fpath, topdown=True):
547                    for elt in files:
548                        path = os.path.join(root, elt)
549
550                        rpath = self.ws.repo.path
551                        if not rpath.endswith('/'):
552                            rpath += '/'
553
554                        path = path.replace(rpath, '', 1)
555                        if path not in tarnames:
556                            return True # In workspace not tar
557            else:
558                if os.path.exists(fpath) and mfile not in tarnames:
559                    return True
560
561        return False
562
563    def cleanup(self):
564        '''Remove backed up workspace metadata'''
565        if os.path.exists(self.bu.backupfile('metadata.tar.gz')):
566            os.unlink(self.bu.backupfile('metadata.tar.gz'))
567
568
569class CdmBackup(object):
570    '''A backup of a given workspace'''
571
572    def __init__(self, ui, ws, name):
573        self.ws = ws
574        self.ui = ui
575        self.backupdir = self._find_backup_dir(name)
576
577        #
578        # The order of instances here controls the order the various operations
579        # are run.
580        #
581        # There's some inherent dependence, in that on restore we need
582        # to restore committed changes prior to uncommitted changes
583        # (as the parent revision of any uncommitted changes is quite
584        # likely to not exist until committed changes are restored).
585        # Metadata restore can happen at any point, but happens last
586        # as a matter of convention.
587        #
588        self.modules = [x(self, ws) for x in [CdmCommittedBackup,
589                                              CdmUncommittedBackup,
590                                              CdmMetadataBackup]]
591
592
593        if os.path.exists(os.path.join(self.backupdir, 'latest')):
594            generation = os.readlink(os.path.join(self.backupdir, 'latest'))
595            self.generation = int(os.path.split(generation)[1])
596        else:
597            self.generation = 0
598
599    def _find_backup_dir(self, name):
600        '''Find the path to an appropriate backup directory based on NAME'''
601        backupdir = None
602        backupbase = None
603
604        if os.path.isabs(name):
605            return name
606
607        if self.ui.config('cdm', 'backupdir'):
608            backupbase = os.path.expanduser(self.ui.config('cdm', 'backupdir'))
609        else:
610            home = None
611
612            try:
613                home = os.getenv('HOME') or pwd.getpwuid(os.getuid()).pw_dir
614            except KeyError:
615                pass                    # Handled anyway
616
617            if not home:
618                raise util.Abort('Could not determine your HOME directory to '
619                                 'find backup path')
620
621            backupbase = os.path.join(home, 'cdm.backup')
622
623        backupdir = os.path.join(backupbase, name)
624
625        # If backupdir exists, it must be a directory.
626        if (os.path.exists(backupdir) and not os.path.isdir(backupdir)):
627            raise util.Abort('%s exists but is not a directory' % backupdir)
628
629        return backupdir
630
631    def backupfile(self, path):
632        '''return full path to backup file FILE at GEN'''
633        return os.path.join(self.backupdir, str(self.generation), path)
634
635    def update_latest(self, gen):
636        '''Update latest symlink to point to the current generation'''
637        linkpath = os.path.join(self.backupdir, 'latest')
638
639        if os.path.lexists(linkpath):
640            os.unlink(linkpath)
641
642        os.symlink(str(gen), linkpath)
643
644    def create_gen(self, gen):
645        '''Create a new backup generation'''
646        try:
647            os.makedirs(os.path.join(self.backupdir, str(gen)))
648            self.update_latest(gen)
649        except EnvironmentError, e:
650            raise util.Abort("Couldn't create backup generation %s: %s" %
651                             (os.path.join(self.backupdir, str(gen)), e))
652
653    def need_backup(self):
654        '''Compare backed up changes to workspace'''
655        #
656        # If there's no current backup generation, or the last backup was
657        # invalid (lacking the dirstate file), we need a backup regardless
658        # of anything else.
659        #
660        if (not self.generation or
661            not os.path.exists(self.backupfile('dirstate'))):
662            return True
663
664        for x in self.modules:
665            if x.need_backup():
666                return True
667
668        return False
669
670    def backup(self):
671        '''Take a backup of the current workspace'''
672
673        if not os.path.exists(self.backupdir):
674            try:
675                os.makedirs(self.backupdir)
676            except EnvironmentError, e:
677                raise util.Abort('Could not create backup directory %s: %s' %
678                                 (self.backupdir, e))
679
680        self.generation += 1
681        self.create_gen(self.generation)
682
683        #
684        # Lock the repo, so the backup can be consistent.  We need the
685        # wlock too to make sure the dirstate parent doesn't change
686        # underneath us.
687        #
688
689        lock = self.ws.repo.lock()
690        wlock = self.ws.repo.lock()
691
692        try:
693            for x in self.modules:
694                x.backup()
695        except Exception, e:
696            if isinstance(e, KeyboardInterrupt):
697                self.ws.ui.warn("Interrupted\n")
698            else:
699                self.ws.ui.warn("Error: %s\n" % e)
700
701                #
702                # If it's not a 'normal' error, we want to print a stack
703                # trace now in case the attempt to remove the partial
704                # backup also fails, and raises a second exception.
705                #
706                if (not isinstance(e, (EnvironmentError, util.Abort))
707                    or self.ws.ui.traceback):
708                    traceback.print_exc()
709
710            for x in self.modules:
711                x.cleanup()
712
713            os.rmdir(os.path.join(self.backupdir, str(self.generation)))
714            self.generation -= 1
715
716            if self.generation != 0:
717                self.update_latest(self.generation)
718            else:
719                os.unlink(os.path.join(self.backupdir, 'latest'))
720
721            raise util.Abort('Backup failed')
722
723    def restore(self, gen=None):
724        '''Restore workspace from backup
725
726        Restores from backup generation GEN (defaulting to the latest)
727        into workspace WS.'''
728
729        wlock = self.ws.repo.wlock()
730        lock = self.ws.repo.lock()
731
732        if not os.path.exists(self.backupdir):
733            raise util.Abort('Backup directory does not exist: %s' %
734                             (self.backupdir))
735
736        if gen:
737            if not os.path.exists(os.path.join(self.backupdir, str(gen))):
738                raise util.Abort('Backup generation does not exist: %s' %
739                                 (os.path.join(self.backupdir, str(gen))))
740            self.generation = int(gen)
741
742        if not self.generation: # This is ok, 0 is not a valid generation
743            raise util.Abort('Backup has no generations: %s' % self.backupdir)
744
745        if not os.path.exists(self.backupfile('dirstate')):
746            raise util.Abort('Backup %s/%s is incomplete (dirstate missing)' %
747                             (self.backupdir, self.generation))
748
749        try:
750            for x in self.modules:
751                x.restore()
752        except util.Abort, e:
753            raise util.Abort('Error restoring workspace:\n'
754                             '%s\n'
755                             'Workspace may be partially restored' % e)
756