1 from os import curdir
2 from os.path import abspath, exists
3 from sets import Set
4 from urlparse import urljoin, urlparse
5 from urllib import pathname2url, url2pathname
6 from urllib2 import URLError
7 from weakref import WeakKeyDictionary, WeakValueDictionary, ref
8
9 from advene.model.consts import _RAISE, PARSER_META_PREFIX
10 from advene.model.backends.exceptions import PackageInUse
11 from advene.model.backends.register import iter_backends
12 import advene.model.backends.sqlite as sqlite_backend
13 from advene.model.core.element import \
14 MEDIA, ANNOTATION, RELATION, TAG, LIST, IMPORT, QUERY, VIEW, RESOURCE
15 from advene.model.core.media import Media, DEFAULT_FOREF
16 from advene.model.core.annotation import Annotation
17 from advene.model.core.relation import Relation
18 from advene.model.core.view import View
19 from advene.model.core.resource import Resource
20 from advene.model.core.tag import Tag
21 from advene.model.core.list import List
22 from advene.model.core.query import Query
23 from advene.model.core.import_ import Import
24 from advene.model.core.all_group import AllGroup
25 from advene.model.core.own_group import OwnGroup
26 from advene.model.core.meta import WithMetaMixin
27 from advene.model.exceptions import \
28 NoClaimingError, NoSuchElementError, UnreachableImportError
29 from advene.model.events import PackageEventDelegate, WithEventsMixin
30 from advene.model.parsers.register import iter_parsers
31 from advene.model.serializers.register import iter_serializers
32 from advene.util.autoproperty import autoproperty
33 from advene.util.files import smart_urlopen
34 from advene.model.tales import tales_path1_function, WithAbsoluteUrlMixin
35
36 _constructor = {
37 MEDIA: "media_factory",
38 ANNOTATION: "annotation_factory",
39 RELATION: "relation_factory",
40 VIEW: "view_factory",
41 RESOURCE: "resource_factory",
42 TAG: "tag_factory",
43 LIST: "list_factory",
44 QUERY: "query_factory",
45 IMPORT: "import_factory",
46 }
47
48 -def _noop(*args, **kw):
50
52 abscurdir = abspath(curdir)
53 abslocal = "file:" + pathname2url(abscurdir) + "/"
54 return urljoin(abslocal, url)
55
56 -class Package(WithMetaMixin, WithEventsMixin, WithAbsoluteUrlMixin, object):
57 """FIXME: missing docstring.
58 """
59
60 annotation_factory = Annotation
61 all_factory = AllGroup
62 import_factory = Import
63 list_factory = List
64 media_factory = Media
65 relation_factory = Relation
66 resource_factory = Resource
67 own_factory = OwnGroup
68 query_factory = Query
69 tag_factory = Tag
70 view_factory = View
71
72 - def __init__(self, url, create=False, readonly=False, force=False):
73 """FIXME: missing docstring.
74
75 @param url: the URL of the package
76 @type url: string
77 @param create: should the package be created ?
78 @type create: boolean
79 @param readonly: should the package be readonly (in the case of loading an existing package) ?
80 @type readonly: boolean
81 @param force: ???
82 @type force: boolean
83 """
84 assert not (create and readonly), "Cannot create a read-only package"
85 self._url = url = _make_absolute(url)
86 self._readonly = readonly
87 self._backend = None
88 self._transient = False
89 self._serializer = None
90 parser = None
91 if create:
92 for b in iter_backends():
93 claims = b.claims_for_create(url)
94 if claims:
95 backend, package_id = b.create(self, force)
96 break
97 elif claims.exception:
98 raise claims.exception
99 else:
100 backend, package_id = self._make_transient_backend()
101 else:
102 for b in iter_backends():
103 claims = b.claims_for_bind(url)
104 if claims:
105 backend, package_id = b.bind(self, force)
106 break
107 elif claims.exception:
108 raise claims.exception
109 else:
110 try:
111 f = smart_urlopen(url)
112 except URLError:
113 raise NoClaimingError("bind %s (URLError)" % url)
114 cmax = 0
115 for p in iter_parsers():
116 c = p.claims_for_parse(f)
117 if c > cmax:
118 cmax = c
119 parser = p
120 if cmax > 0:
121 self._serializer = parser.SERIALIZER
122 backend, package_id = self._make_transient_backend()
123 else:
124 f.close()
125 raise NoClaimingError("bind %s" % url)
126
127 self._backend = backend
128 self._id = package_id
129 self._elements = WeakValueDictionary()
130 self._heavy_elements = Set()
131 self._own_wref = lambda: None
132 self._all_wref = lambda: None
133 self._uri = None
134
135
136
137 self._imports_dict = imports_dict = {}
138
139
140 self._importers = WeakKeyDictionary()
141
142
143 self._backends_dict = None
144
145
146
147
148
149 if parser:
150 parser.parse_into(f, self)
151 f.close()
152
153
154
155 package_class = self.__class__
156 for _, _, iid, url, uri in backend.iter_imports((package_id,)):
157 p = None
158 try:
159 p = package_class(url)
160 except NoClaimingError:
161 if uri:
162 try:
163 p = package_class(url)
164 except NoClaimingError:
165 pass
166 except PackageInUse, e:
167 if isinstance(e.message, package_class):
168 p = e.message
169 if p is None:
170 pass
171
172 else:
173 if uri != p._uri:
174 pass
175
176 p._importers[self] = iid
177 imports_dict[iid] = p
178
179 self._update_backends_dict(_firsttime=True)
180
186
188 """FIXME: missing docstring.
189 """
190 claimed = False
191 i = 0
192 while not claimed:
193 url = "sqlite::memory:;transient-%s" % i
194 i += 1
195 claimed = sqlite_backend.claims_for_create(url)
196 self._transient = True
197 return sqlite_backend.create(self, url=url)
198
200 """FIXME: missing docstring.
201 """
202 def signature(d):
203 return dict( (k, v.keys()) for k, v in d.items() )
204 if not _firsttime:
205 oldsig = signature(self._backends_dict)
206 self._backends_dict = None
207 in_construction = {}
208
209 visited = {}; queue = [self,]
210 while queue:
211 p = queue[-1]
212 visited[p] = 1
213 if p._backends_dict is not None:
214
215 for backend, ids in p._backends_dict.iteritems():
216 d = in_construction.get(backend)
217 if d is None:
218 d = in_construction[backend] = WeakValueDictionary()
219 for i,q in ids.items():
220 if i not in d:
221 d[i] = q
222 queue.pop(-1)
223 else:
224
225 d = in_construction.get(p._backend)
226 if d is None:
227 d = in_construction[p._backend] = WeakValueDictionary()
228 d[p._id] = p
229
230 for q in p._imports_dict.itervalues():
231 if q is not None and q not in visited:
232 queue.append(q)
233 break
234 else:
235 queue.pop(-1)
236 self._backends_dict = in_construction
237 if not _firsttime:
238 newsig = signature(self._backends_dict)
239 if oldsig != newsig:
240 for p in self._importers:
241 p._update_backends_dict()
242
244 """
245 Return a dict whose keys are backends, and whose values are dicts
246 whose keys are package ids, and whose keys are packages. This
247 dict contains all the direct importers of this package, plus this
248 package itself.
249 """
250
251
252 r = {}
253 for p, iid in list(self._importers.iteritems()) + [(self, ""),]:
254 d = r.get(p._backend)
255 if d is None:
256 d = r[p._backend] = {}
257 d[p._id] = p
258 return r
259
261 """Free all external resources used by the package's backend.
262
263 It is an error to close a package that is imported by another one,
264 unless they are part of an import cycle. In the latter case, this
265 package will be closed, and the other packages in the cycle will
266 be closed as well.
267
268 It is an error to use a package or any of its elements or attributes
269 when the package has been closed. The behaviour is undefined.
270 """
271 imp = self._importers
272 if not imp:
273 self._do_close()
274 self._finish_close()
275 else:
276 cycles = {}
277 for be, pdict in self._backends_dict.iteritems():
278 for pid, p in pdict.items():
279 if self._id in p._backends_dict.get(self._backend, ()):
280 cycles[p] = True
281 for i in imp:
282 if i not in cycles:
283 raise ValueError(
284 "Can not close, package is imported by <%s>" %
285 (i.uri or i.url,)
286 )
287 for p in cycles:
288 p._do_close()
289 for p in cycles:
290 p._finish_close()
291
293 if self._transient:
294 self._backend.delete(self._id)
295 else:
296 self._backend.close(self._id)
297 self._backend = None
298 self.emit("package-closed", self._url, self._uri)
299
301 """FIXME: missing docstring.
302 """
303
304 self._backends_dict = None
305 for p in self._imports_dict.itervalues():
306 if p is not None:
307 p._importers.pop(self, None)
308 self._imports_dict = None
309
310 - def save(self, serializer=None):
311 """Save the package to disk if its URL is in the "file:" scheme.
312
313 A specific serializer module can be provided, else if the package was
314 parsed and the parser had a corresponding serializer, that one will be
315 used; else, the extension of the filename will be used to guess the
316 serializer to use.
317
318 Note that the file will be silently erased if it already exists.
319 """
320 p = urlparse(self._url)
321 if p.scheme not in ('file', ''):
322 raise ValueError("Can not save to URL %s" % self._url)
323 filename = url2pathname(p.path)
324
325 self.save_as(filename, serializer=serializer or self._serializer,
326 change_url=True, erase=True)
327
328
329 - def save_as(self, url, change_url=False, serializer=None, erase=False):
330 """
331 Save the package under the given URL (if it is in the 'file:' scheme).
332
333 If `change_url` is set, the URL of the package will be modified to the
334 corresponding ``file:`` URL.
335
336 A specific serializer module can be provided, else the extension of the
337 filename will be used to guess the serializer to use.
338
339 Note that if the file exists, an exception will be raised.
340 """
341 p = urlparse(url)
342 if p.scheme not in ('file', ''):
343 raise ValueError("Can not save to URL %s" % url)
344 filename = url2pathname(p.path)
345
346 if exists(filename) and not erase:
347 raise Exception("File already exists %s" % filename)
348
349 s = serializer
350 if s is None:
351 for s in iter_serializers():
352 if filename.endswith(s.EXTENSION):
353 break
354 else:
355 raise Exception("Can not guess correct serializer for %s" %
356 filename)
357
358 f = open(filename, "w")
359 s.serialize_to(self, f)
360 f.close()
361
362 if change_url:
363 filename = abspath(filename)
364 self._url = url = "file:" + pathname2url(filename)
365 self._backend.update_url(self._id, self._url)
366 self._serializer = serializer
367
368 @autoproperty
370 """
371 The URL from which this package has been fetched.
372 """
373 return self._url
374
375 @autoproperty
377 return self._readonly
378
379 @autoproperty
381 """
382 The URI identifying this package.
383
384 It may be different from the URL from which the package has actually
385 been fetched.
386 """
387 r = self._uri
388 if r is None:
389 r = self._uri = self._backend.get_uri(self._id)
390 return r
391
392 @autoproperty
403
404 @autoproperty
406 r = self._own_wref()
407 if r is None:
408 r = self.own_factory(self)
409 self._own_wref = ref(r)
410 return r
411
412 @autoproperty
414 r = self._all_wref()
415 if r is None:
416 r = self.all_factory(self)
417 self._all_wref = ref(r)
418 return r
419
420 @property
422 return self._backend is None
423
424
425
427 if element_type is None:
428 return id in self._elements \
429 or self._backend.has_element(self._id, id)
430 else:
431 e = self._elements.get(id)
432 return (e is not None and e.ADVENE_TYPE == element_type) \
433 or self._backend.has_element(self._id, id, element_type)
434
436 """Get the element with the given id-ref or uri-ref.
437
438 If the element does not exist, an exception is raised (see below)
439 unless ``default`` is provided, in which case its value is returned.
440
441 If 'id' contains a '#', it is assumed to be a URI-ref, else it is
442 assumed to be an ID-ref. In both cases, all imported packages are
443 searched for the element
444
445 An `UnreachableImportError` is raised if the given id involves an
446 nonexistant or unreachable import. A `NoSuchElementError` is raised if
447 the last item of the id-ref is not the id of an element in the
448 corresponding package.
449
450 Note that packages are also similar to python dictionaries, so
451 `__getitem__` and `get` can also be used to get elements.
452 """
453
454
455
456
457
458
459
460
461
462
463
464 if not isinstance(id, basestring):
465 tuple = id
466 id = tuple[2]
467 else:
468 tuple = None
469 sharp = id.find("#")
470 if sharp >= 0:
471 return self.get_element_by_uriref(id, default)
472 colon = id.find(":")
473 if colon <= 0:
474 return self._get_own_element(id, tuple, default)
475 else:
476 assert tuple is None
477 imp = id[:colon]
478 pkg = self._imports_dict.get(imp)
479 if pkg is None:
480 if default is _RAISE:
481 raise UnreachableImportError(imp)
482 else:
483 return default
484 else:
485 return pkg.get_element(id[colon+1:], default)
486
487 @tales_path1_function
488 - def get(self, id, default=None):
490
492 """Get the element with the given uri-ref.
493
494 If the element does not exist, an exception is raised (see below)
495 unless ``default`` is provided, in which case its value is returned.
496
497 An `UnreachableImportError` is raised if the given id involves an
498 nonexistant or unreachable import. A `NoSuchElementError` is raised if
499 the last item of the id-ref is not the id of an element in the
500 corresponding package.
501 """
502 sharp = uriref.index("#")
503 package_uri, id = uriref[:sharp], uriref[sharp+1:]
504 for _, ps in self._backends_dict.iteritems():
505 for p in ps.itervalues():
506 if package_uri == p.uri or package_uri == p.url:
507 return p.get_element(id, default)
508 raise NoSuchElementError(uriref)
509
510 __getitem__ = get_element
511
513 """Get the element whose id is given from the own package's elements.
514
515 Id may be a simple id or a path id.
516
517 If necessary, it is made from backend data, then stored (as a weak ref)
518 in self._elements to prevent several instances of the same element to
519 be produced.
520 """
521 r = self._elements.get(id)
522 if r is None:
523 c = tuple or self._backend.get_element(self._id, id)
524 if c is None:
525 if default is _RAISE:
526 raise NoSuchElementError(id)
527 r = default
528 else:
529 type, init = c[0], c[2:]
530 factory = getattr(self, _constructor[type])
531 r = factory.instantiate(self, *init)
532
533 return r
534
536 """
537 Return True iff element is owned or directly imported by this package.
538
539 element can be either an instance of PackageElement or an id-ref.
540 Note that if element is the id-ref of an imported element, its
541 existence in the imported package is *not* checked (but it is checked
542 that the import exists).
543 """
544 if hasattr(element, "_owner"):
545 o = element._owner
546 return o is self or o in self._imports_dict.values()
547 else:
548 path = _split_idref(unicode(element))
549 if len(path) > 2:
550 return False
551 elif len(path) == 2:
552 return self.has_element(path[0], IMPORT)
553 else:
554 return self.has_element(path[0])
555
557 """Compute an id-ref in this package for an element.
558
559 The element is identified by ``id`` in the package ``pkg``. It is of
560 course assumed that pkg is imported by this package.
561
562 See also `PackageElement.make_id_in`.
563 """
564 if self is pkg:
565 return id
566
567
568 queue = self._imports_dict.items()
569 current = 0
570 visited = {self:True}
571 parent = {}
572 found = False
573 while not found and current < len(queue):
574 prefix,p = queue[current]
575 if p is pkg:
576 found = True
577 else:
578 if p is not None:
579 visited[p] = True
580 for prefix2,p2 in p._imports_dict.iteritems():
581 if p2 not in visited:
582 queue.append((prefix2,p2))
583 parent[(prefix2,p2)] = (prefix,p)
584 current += 1
585 if not found:
586 raise ValueError("Element is not reachable from that package")
587 r = id
588 c = queue[current]
589 while c is not None:
590 r = "%s:%s" % (c[0], r)
591 c = parent.get(c)
592 return r
593
595
596
597
598 raise ValueError("not iterable, use X.own or X.all instead")
599
600
601
608
609 - def create_annotation(self, id, media, begin, end,
610 mimetype, model=None, url=""):
617
618 - def create_relation(self, id, mimetype="x-advene/none", model=None,
619 url="", members=()):
620 """FIXME: missing docstring.
621 """
622 r = self.relation_factory.create_new(self, id, mimetype, model, url, members)
623 self.emit("created::relation", r)
624 return r
625
626 - def create_view(self, id, mimetype, model=None, url=""):
632
634 """FIXME: missing docstring.
635 """
636 r = self.resource_factory.create_new(self, id, mimetype, model, url)
637 self.emit("created::resource", r)
638 return r
639
641 """FIXME: missing docstring.
642 """
643 r = self.tag_factory.create_new(self, id)
644 self.emit("created::tag", r)
645 return r
646
648 """FIXME: missing docstring.
649 """
650 r = self.list_factory.create_new(self, id, items)
651 self.emit("created::list", r)
652 return r
653
655 """FIXME: missing docstring.
656 """
657 assert not self.has_element(id), "The identifier %s already exists" % id
658 r = self.query_factory.create_new(self, id, mimetype, model, url)
659 self.emit("created::query", r)
660 return r
661
663 """FIXME: missing docstring.
664 """
665 r = self.import_factory.create_new(self, id, package)
666 self.emit("created::import", r)
667 return r
668
670 """
671 As it name implies, this method is stricly reserved to parsers for
672 creating imports without actually loading them. It *must not* be
673 called elsewhere (it would corrupt the package w.r.t. imports).
674 """
675 self._backend.create_import(self._id, id, url, uri)
676 r = self.get(id)
677 return r
678
679
680
682 """
683 Associate the given element to the given tag on behalf of this package.
684
685 `element` must normally be a PackageElement instance and `tag` a TAG
686 instance. In the case one of them is an imported element, the id-ref
687 can actually be given instead of the actual element, but this should be
688 used only in situation where robustness to unreachable elements is
689 desirable (e.g. parsers).
690 """
691 assert self._can_reference(element), element
692 assert self._can_reference(tag), tag
693 assert getattr(tag, "ADVENE_TYPE", TAG) == TAG, "The tag should be a Tag"
694
695 elt_owner = getattr(element, "_owner", None)
696 if elt_owner:
697 if elt_owner is self:
698 id_e = element._id
699 else:
700 id_e = element.make_id_in(self)
701 else:
702 assert element.find(":") > 0, "Expected *strict* id-ref"
703 id_e = unicode(element)
704 tag_owner = getattr(tag, "_owner", None)
705 if tag_owner:
706 if tag_owner is self:
707 id_t = tag._id
708 else:
709 id_t = tag.make_id_in(self)
710 else:
711 assert tag.find(":") > 0, "Expected *strict* id-ref"
712 id_t = unicode(tag)
713
714 self._backend.associate_tag(self._id, id_e, id_t)
715 getattr(element, "emit", _noop)("added-tag", tag)
716 getattr(tag, "emit", _noop)("added", element)
717
719 """
720 Dissociate the given element to the given tag on behalf of this package.
721 """
722 assert self._can_reference(element), element
723 assert self._can_reference(tag), tag
724 assert getattr(tag, "ADVENE_TYPE", TAG) == TAG, "The tag should be a Tag"
725
726 elt_owner = getattr(element, "_owner", None)
727 if elt_owner:
728 if elt_owner is self:
729 id_e = element._id
730 else:
731 id_e = element.make_id_in(self)
732 else:
733 assert element.find(":") > 0, "Expected *strict* id-ref"
734 id_e = unicode(element)
735 tag_owner = getattr(tag, "_owner", None)
736 if tag_owner:
737 if tag_owner is self:
738 id_t = tag._id
739 else:
740 id_t = tag.make_id_in(self)
741 else:
742 assert tag.find(":") > 0, "Expected *strict* id-ref"
743 id_t = unicode(tag)
744
745 self._backend.dissociate_tag(self._id, id_e, id_t)
746 getattr(element, "emit", _noop)("removed-tag", tag)
747 getattr(tag, "emit", _noop)("removed", element)
748
749
750
751
752
753
754
756 """
757 Return a dict representing the parser-meta:namespaces metadata, with
758 URIs as keys and prefixes as values.
759
760 Note that changing this dict does not affect the metadata. For this,
761 use ``_set_namespaces_with_dict``.
762 """
763 r = {}
764 prefixes = self.get_meta(PARSER_META_PREFIX+"namespaces" , "")
765 for line in prefixes.split("\n"):
766 if line:
767 prefix, uri = line.split(" ")
768 r[uri] = prefix
769 return r
770
772 """
773 Set the parser-meta:namespaces metadata with a dict like the one
774 returned by ``_get_namespaces_as_dict``.
775 """
776 s = "\n".join( "%s %s" % (prefix, uri)
777 for uri, prefix in d.iteritems() )
778 self.set_meta(PARSER_META_PREFIX+"namespaces", s)
779
780
781
783 """
784 Used by `WithAbsoluteUrlMixin`
785 """
786 a=aliases.get(self, None)
787 if a is not None:
788 return a
789
790
791 if self.uri:
792 kw = {"uri": self.uri}
793 else:
794 kw = {"url": self.url}
795 for p, a in aliases.iteritems():
796 for imp in p.all.iter_imports(**kw):
797 return "%s/%s/package" % (a, imp.make_id_in(p))
798
799 self._absolute_url_fail("Cannot find reference for package %s" % self.uri)
800
801 @property
804
805 @property
808
809 @property
812
813 @property
816
817 @property
820
821 @property
824
825 @property
828
829 @property
832
833 @property
836
839 """
840 Split an ID-ref into a list of atomic IDs.
841 """
842 path1 = idref.split("::")
843 if len(path1) == 1:
844 return idref.split(":")
845 elif path1[0]:
846 return path1[0].split(":") + [":%s" % path1[1]]
847 else:
848 return [ idref, ]
849
850