1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 from twisted.internet import defer
23
24 from flumotion.common import log
25 from flumotion.component.misc.httpserver import cachemanager
26 from flumotion.component.misc.httpserver import cachestats
27 from flumotion.component.misc.httpserver import localpath
28 from flumotion.component.misc.httpserver.httpcached import http_client
29 from flumotion.component.misc.httpserver.httpcached import http_utils
30 from flumotion.component.misc.httpserver.httpcached import request_manager
31 from flumotion.component.misc.httpserver.httpcached import resource_manager
32 from flumotion.component.misc.httpserver.httpcached import server_selection
33 from flumotion.component.misc.httpserver.httpcached import strategy_basic
34
35
36 LOG_CATEGORY = "filereader-httpcached"
37
38 DEFAULT_CACHE_TTL = 5*60
39 DEFAULT_DNS_REFRESH = 60
40 DEFAULT_VIRTUAL_PORT = 80
41 DEFAULT_VIRTUAL_PATH = ""
42 DEFAULT_SERVER_PORT = 3128
43 DEFAULT_PROXY_PRIORITY = 1
44 DEFAULT_CONN_TIMEOUT = 2
45 DEFAULT_IDLE_TIMEOUT = 5
46
47
49 """
50 Offers a file-like interface to streams retrieved using HTTP.
51 It supports:
52 - Local caching with TTL expiration, and cooperative managment.
53 - Load-balanced HTTP servers with priority level (fall-back).
54 - More than one IP by server hostname with periodic DNS refresh.
55 - Connection resuming if HTTP connection got disconnected.
56 """
57
58 logCategory = LOG_CATEGORY
59
61 props = args['properties']
62
63 cacheDir = props.get('cache-dir')
64 cacheSizeInMB = props.get('cache-size')
65 if cacheSizeInMB is not None:
66 cacheSize = cacheSizeInMB * 10 ** 6
67 else:
68 cacheSize = None
69 cleanupEnabled = props.get('cleanup-enabled')
70 cleanupHighWatermark = props.get('cleanup-high-watermark')
71 cleanupLowWatermark = props.get('cleanup-low-watermark')
72
73 self.virtualHost = props.get('virtual-hostname')
74 self.virtualPort = props.get('virtual-port', DEFAULT_VIRTUAL_PORT)
75 self.virtualPath = props.get('virtual-path', DEFAULT_VIRTUAL_PATH)
76 dnsRefresh = props.get('dns-refresh-period', DEFAULT_DNS_REFRESH)
77 servers = props.get('http-server')
78 compat_servers = props.get('http-server-old')
79
80 self.stats = cachestats.CacheStatistics()
81
82 self.cachemgr = cachemanager.CacheManager(self.stats,
83 cacheDir, cacheSize,
84 cleanupEnabled,
85 cleanupHighWatermark,
86 cleanupLowWatermark,
87 self.virtualHost)
88
89 selector = server_selection.ServerSelector(dnsRefresh)
90
91 if not (servers or compat_servers):
92 selector.addServer(self.virtualHost, self.virtualPort)
93 else:
94 if compat_servers:
95
96 for hostname in compat_servers:
97 if '#' in hostname:
98 hostname, priostr = hostname.split('#', 1)
99 priority = int(priostr)
100 else:
101 priority = DEFAULT_PROXY_PRIORITY
102 if ':' in hostname:
103 hostname, portstr = hostname.split(':', 1)
104 port = int(portstr)
105 else:
106 port = DEFAULT_SERVER_PORT
107 selector.addServer(hostname, port, priority)
108
109
110 if servers:
111
112 for serverProps in servers:
113 hostname = serverProps.get('hostname')
114 port = serverProps.get('port', DEFAULT_SERVER_PORT)
115 priority = serverProps.get('priority',
116 DEFAULT_PROXY_PRIORITY)
117 selector.addServer(hostname, port, priority)
118
119 connTimeout = props.get('connection-timeout', DEFAULT_CONN_TIMEOUT)
120 idleTimeout = props.get('idle-timeout', DEFAULT_IDLE_TIMEOUT)
121
122 client = http_client.StreamRequester(connTimeout, idleTimeout)
123
124 reqmgr = request_manager.RequestManager(selector, client)
125
126 cacheTTL = props.get('cache-ttl', DEFAULT_CACHE_TTL)
127
128 self.strategy = strategy_basic.CachingStrategy(self.cachemgr,
129 reqmgr, cacheTTL)
130
131 self.resmgr = resource_manager.ResourceManager(self.strategy,
132 self.stats)
133
135 d = defer.Deferred()
136 d.addCallback(lambda _: self.cachemgr.setUp())
137 d.addCallback(lambda _: self.strategy.setup())
138 d.addCallback(lambda _: self)
139 d.callback(None)
140 return d
141
143 d = defer.Deferred()
144 d.addCallback(lambda _: self.strategy.cleanup())
145 d.addCallback(lambda _: self)
146 d.callback(None)
147 return d
148
149 - def open(self, path):
154