Annotation of /trunk/pkgtools/svn-cleanup-pkg-tree-sql.sh
Parent Directory | Revision Log
Revision 2108 -
(hide annotations)
(download)
(as text)
Mon May 13 11:37:21 2013 UTC (11 years, 4 months ago) by niro
File MIME type: application/x-sh
File size: 7720 byte(s)
Mon May 13 11:37:21 2013 UTC (11 years, 4 months ago) by niro
File MIME type: application/x-sh
File size: 7720 byte(s)
added cleanup script with sqlite support with significant speed improvements
1 | niro | 2108 | #!/bin/bash |
2 | |||
3 | # export default C locale | ||
4 | export LC_ALL=C | ||
5 | |||
6 | MY_REPOS=( "branches/R11-stable" "trunk" ) | ||
7 | MY_ARCHES=( "x86_64" "i686" ) | ||
8 | MY_DISTRO_PKG_BASE="/var/www/htdocs/magellan/R11" | ||
9 | MY_DISTRO_HTTP_PKG_BASE="http://magellan-linux.de/magellan/R11" | ||
10 | MY_SSHLOGIN_BASE="tjoke@magellan-linux.de" | ||
11 | MY_MAGE_FEATURES=( "pkgdistrotag" ) | ||
12 | MY_MAGE_FEATURES_i686=( "srcpkgbuild" ) | ||
13 | |||
14 | # decide wether to use http method or not | ||
15 | USE_HTTP=0 | ||
16 | if [[ $1 = --http ]] | ||
17 | then | ||
18 | USE_HTTP=1 | ||
19 | shift | ||
20 | fi | ||
21 | |||
22 | MY_SVN_BASE="$1" | ||
23 | if [[ -z ${MY_SVN_BASE} ]] | ||
24 | then | ||
25 | MY_SVN_BASE="$(mktemp -d -t tree-cleanup.XXXXXXXXXX)" | ||
26 | fi | ||
27 | MY_SVN_OPTS="" | ||
28 | MY_SVN_ROOT="svn://cvs.magellan-linux.de" | ||
29 | |||
30 | DBFILE="${MY_SVN_BASE}/pkgdb" | ||
31 | |||
32 | sql() | ||
33 | { | ||
34 | local sqlcmd="$*" | ||
35 | [[ -z ${sqlcmd} ]] && die "no sqlcmd given." | ||
36 | |||
37 | sqlite3 -nullvalue 'NULL' -list -separator '|' ${DBFILE} << EOF || die "error running '$@'" | ||
38 | ${sqlcmd}; | ||
39 | EOF | ||
40 | } | ||
41 | |||
42 | # load_arch_values ARCH | ||
43 | load_arch_values() | ||
44 | { | ||
45 | local arch="$1" | ||
46 | |||
47 | MAGE_FEATURES=( ${MY_MAGE_FEATURES[*]} ) | ||
48 | MAGE_FEATURES+=( $(eval echo \${MY_MAGE_FEATURES_${arch}[*]}) ) | ||
49 | export MAGE_FEATURES | ||
50 | |||
51 | export ARCH="${arch}" | ||
52 | export CHOST="${arch}-pc-linux-gnu" | ||
53 | export PACKAGES_SERVER_PATH="packages/${arch}" | ||
54 | export PKGDIR="${MY_DISTRO_PKG_BASE}/packages/${arch}/" | ||
55 | export HTTP_PKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/${arch}/" | ||
56 | export SRCPKGDIR="${MY_DISTRO_PKG_BASE}/packages/src/" | ||
57 | export HTTP_SRCPKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/src/" | ||
58 | } | ||
59 | |||
60 | # generate_pkg_file_list REPO | ||
61 | generate_pkg_file_list() | ||
62 | { | ||
63 | local repo="$1" | ||
64 | local smage | ||
65 | local arch | ||
66 | local _findlist | ||
67 | local subpackage | ||
68 | export SMAGESCRIPTSDIR="${repo}" | ||
69 | |||
70 | _findlist=$(find ${repo} -name \*.${SMAGESUFFIX} | sort -u) | ||
71 | for arch in ${MY_ARCHES[*]} | ||
72 | do | ||
73 | load_arch_values ${arch} | ||
74 | load_mage_features | ||
75 | |||
76 | for smage in ${_findlist} | ||
77 | do | ||
78 | echo "Processing ${smage}" | ||
79 | smagesource ${smage} &> /dev/null || die | ||
80 | |||
81 | # honor split-packages | ||
82 | if [[ -n ${SPLIT_PACKAGES} ]] | ||
83 | then | ||
84 | local split_pkg_base="${PNAME}" | ||
85 | split_save_variables | ||
86 | for subpackage in ${SPLIT_PACKAGES} | ||
87 | do | ||
88 | export PNAME="${subpackage}" | ||
89 | split_info_${subpackage} | ||
90 | # fix PCATEGORIE -> PCAT | ||
91 | if [[ ! -z ${PCATEGORIE} ]] | ||
92 | then | ||
93 | PCAT="${PCATEGORIE}" | ||
94 | unset PCATEGORIE | ||
95 | fi | ||
96 | |||
97 | # echo "DEBUG: split_package()" | ||
98 | # echo "DEBUG: PNAME='${PNAME}'" | ||
99 | # echo "DEBUG: PVER='${PVER}'" | ||
100 | # echo "DEBUG: PBUILD='${PBUILD}'" | ||
101 | # echo "DEBUG: SMAGESCRIPTSDIR='${SMAGESCRIPTSDIR}'" | ||
102 | # echo "DEBUG: DISTROTAG='${DISTROTAG}'" | ||
103 | # echo "DEBUG: print_distrotag()='$(print_distrotag)'" | ||
104 | # echo "DEBUG: ARCH='${ARCH}'" | ||
105 | # echo "DEBUG: CHOST='${CHOST}'" | ||
106 | # echo "DEBUG: PACKAGES_SERVER_PATH='${PACKAGES_SERVER_PATH}'" | ||
107 | # echo "DEBUG: PKGDIR='${PKGDIR}'" | ||
108 | # echo "DEBUG: SRCPKGDIR='${SRCPKGDIR}'" | ||
109 | |||
110 | sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values( | ||
111 | '${PNAME}', | ||
112 | '${PVER}', | ||
113 | '${PBUILD}', | ||
114 | '${ARCH}', | ||
115 | '$(print_distrotag)', | ||
116 | '${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')" | ||
117 | |||
118 | # restore smage environment | ||
119 | split_restore_variables | ||
120 | done | ||
121 | # unset all saved smage variables | ||
122 | split_unset_variables | ||
123 | unset split_pkg_base | ||
124 | else | ||
125 | sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values( | ||
126 | '${PNAME}', | ||
127 | '${PVER}', | ||
128 | '${PBUILD}', | ||
129 | '${ARCH}', | ||
130 | '$(print_distrotag)', | ||
131 | '${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')" | ||
132 | fi | ||
133 | if mqueryfeature "srcpkgbuild" | ||
134 | then | ||
135 | sql "insert or ignore into srcpkgs(pname, pver, pbuild, filename ) values( | ||
136 | '${PNAME}', | ||
137 | '${PVER}', | ||
138 | '${PBUILD}', | ||
139 | '${PNAME}${target}-${PVER}-${PBUILD}.${SRCPKGSUFFIX}')" | ||
140 | fi | ||
141 | unset PNAME | ||
142 | unset PVER | ||
143 | unset PBUILD | ||
144 | unset SPLIT_PACKAGES | ||
145 | done | ||
146 | done | ||
147 | } | ||
148 | |||
149 | # check_pkglist TABLE PKGFILE | ||
150 | check_pkglist() | ||
151 | { | ||
152 | local table="$1" | ||
153 | local pkg="$2" | ||
154 | |||
155 | if [[ -z $(sql "select filename from ${table} where filename='${pkg}'") ]] | ||
156 | then | ||
157 | return 1 | ||
158 | fi | ||
159 | return 0 | ||
160 | } | ||
161 | |||
162 | # checkout svn sources for mage and smage repos | ||
163 | export SVNROOT="${MY_SVN_ROOT}" | ||
164 | [[ ! -d ${MY_SVN_BASE} ]] && install -d ${MY_SVN_BASE} | ||
165 | for module in magellan-source/trunk/mage ${MY_REPOS[*]} | ||
166 | do | ||
167 | case ${module} in | ||
168 | magellan-source/*) repo="${module}"; codir=mage-src ;; | ||
169 | *) repo="smage/${module}"; codir="smage/${module}" ;; | ||
170 | esac | ||
171 | |||
172 | if [ ! -d ${MY_SVN_BASE}/${module}/.svn ] | ||
173 | then | ||
174 | ( cd ${MY_SVN_BASE}; svn checkout ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} ) | ||
175 | else | ||
176 | ( cd ${MY_SVN_BASE}; svn update ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} ) | ||
177 | fi | ||
178 | |||
179 | # exclude deprecated | ||
180 | if [ -d ${MY_SVN_BASE}/${codir}/deprecated ] | ||
181 | then | ||
182 | echo "deleting ${MY_SVN_BASE}/${codir}/deprecated" | ||
183 | rm -r ${MY_SVN_BASE}/${codir}/deprecated | ||
184 | fi | ||
185 | done | ||
186 | |||
187 | # include functions from fresh mage checkout | ||
188 | source ${MY_SVN_BASE}/mage-src/usr/lib/mage/mage.rc.global.in | ||
189 | export MLIBDIR="${MY_SVN_BASE}/mage-src/usr/lib/mage" | ||
190 | source ${MLIBDIR}/mage4.functions.sh | ||
191 | source ${MLIBDIR}/smage2.functions.sh | ||
192 | |||
193 | sql "create table IF NOT EXISTS pkgs(id integer primary key, | ||
194 | pname text, | ||
195 | pver text, | ||
196 | pbuild text, | ||
197 | arch text, | ||
198 | distrotag text, | ||
199 | filename text unique)" | ||
200 | sql "create table IF NOT EXISTS srcpkgs(id integer primary key, | ||
201 | pname text, | ||
202 | pver text, | ||
203 | pbuild text, | ||
204 | filename text unique)" | ||
205 | |||
206 | for repo in ${MY_REPOS[*]} | ||
207 | do | ||
208 | generate_pkg_file_list "${MY_SVN_BASE}/smage/${repo}" | ||
209 | done | ||
210 | |||
211 | sql "select filename from pkgs" | ||
212 | sql "select filename from srcpkgs" | ||
213 | |||
214 | # initialize lists | ||
215 | if [[ ${USE_HTTP} = 1 ]] | ||
216 | then | ||
217 | :> ${MY_SVN_BASE}/pkglist | ||
218 | :> ${MY_SVN_BASE}/srcpkglist | ||
219 | fi | ||
220 | |||
221 | for arch in ${MY_ARCHES[*]} | ||
222 | do | ||
223 | load_arch_values "${arch}" | ||
224 | load_mage_features | ||
225 | |||
226 | if [[ ${USE_HTTP} = 1 ]] | ||
227 | then | ||
228 | _filelist=$(lynx -dump -listonly ${HTTP_PKGDIR} | grep ".${PKGSUFFIX}" | sed "s:.*${PACKAGES_SERVER_PATH}/\(.*.${PKGSUFFIX}\):\1:") | ||
229 | else | ||
230 | _filelist=$(find ${PKGDIR} -name \*.${PKGSUFFIX} -printf '%f\n' | sort) | ||
231 | fi | ||
232 | |||
233 | if [[ ${USE_HTTP} = 1 ]] | ||
234 | then | ||
235 | echo "install -d ${PKGDIR}/_deprecated" >> ${MY_SVN_BASE}/pkglist | ||
236 | else | ||
237 | if [ ! -d ${PKGDIR}/_deprecated ] | ||
238 | then | ||
239 | install -d ${PKGDIR}/_deprecated || die "install ${PKGDIR}/${arch}/_deprecated" | ||
240 | fi | ||
241 | fi | ||
242 | for file in ${_filelist} | ||
243 | do | ||
244 | if check_pkglist pkgs "${file}" | ||
245 | then | ||
246 | echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}" | ||
247 | else | ||
248 | echo -e "${COLRED}[d] - ${file}${COLDEFAULT}" | ||
249 | if [[ ${USE_HTTP} = 1 ]] | ||
250 | then | ||
251 | echo "mv -v ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ " >> ${MY_SVN_BASE}/pkglist | ||
252 | else | ||
253 | mv ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ || die "move ${PKGDIR}/${file} -> _deprecated" | ||
254 | fi | ||
255 | fi | ||
256 | done | ||
257 | if mqueryfeature "srcpkgbuild" | ||
258 | then | ||
259 | if [[ ${USE_HTTP} = 1 ]] | ||
260 | then | ||
261 | _filelist=$(lynx -dump -listonly ${HTTP_SRCPKGDIR} | grep ".${SRCPKGSUFFIX}" | sed "s:.*packages/src/\(.*.${SRCPKGSUFFIX}\):\1:") | ||
262 | else | ||
263 | _filelist=$(find ${SRCPKGDIR} -name \*.${SRCPKGSUFFIX} -printf '%f\n' | sort) | ||
264 | fi | ||
265 | |||
266 | if [[ ${USE_HTTP} = 1 ]] | ||
267 | then | ||
268 | echo "install -d ${SRCPKGDIR}/_deprecated" >> ${MY_SVN_BASE}/srcpkglist | ||
269 | else | ||
270 | if [ ! -d ${SRCPKGDIR}/_deprecated ] | ||
271 | then | ||
272 | install -d ${SRCPKGDIR}/_deprecated || die "install ${SRCPKGDIR}/_deprecated" | ||
273 | fi | ||
274 | fi | ||
275 | for file in ${_filelist} | ||
276 | do | ||
277 | if check_pkglist srcpkgs "${file}" | ||
278 | then | ||
279 | echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}" | ||
280 | else | ||
281 | echo -e "${COLRED}[d] - ${file}${COLDEFAULT}" | ||
282 | if [[ ${USE_HTTP} = 1 ]] | ||
283 | then | ||
284 | echo "mv -v ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/" >> ${MY_SVN_BASE}/srcpkglist | ||
285 | else | ||
286 | mv ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/ || die "move ${SRCPKGDIR}/${file} -> _deprecated" | ||
287 | fi | ||
288 | fi | ||
289 | done | ||
290 | fi | ||
291 | done | ||
292 | |||
293 | if [[ ${USE_HTTP} = 1 ]] | ||
294 | then | ||
295 | for list in pkglist srcpkglist | ||
296 | do | ||
297 | if [[ -f ${MY_SVN_BASE}/${list} ]] | ||
298 | then | ||
299 | scp ${MY_SVN_BASE}/${list} ${MY_SSHLOGIN_BASE}:~ | ||
300 | ssh ${MY_SSHLOGIN_BASE} bash ~/${list} | ||
301 | fi | ||
302 | done | ||
303 | fi |