Browse code

tdnf: tdnf makecache and --refresh command updates.

Previously tdnf makecache and --refresh always downloads repodata irrespective
of changes in repodata. This change uses shasum comaparison to detect if there
is any change in repodata and downloads repodata only if there is any
change in repodata.

Change-Id: Id4f2c70f1f68a62f928f53d2865dc72aae7bc19f
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/6817
Tested-by: michellew <michellew@vmware.com>
Reviewed-by: Keerthana K <keerthanak@vmware.com>

Keerthana K authored on 2019/03/04 15:22:03
Showing 2 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,321 @@
0
+From e6012ec141c05b20928263517d51b5b0a88ae272 Mon Sep 17 00:00:00 2001
1
+From: Keerthana K <keerthanak@vmware.com>
2
+Date: Fri, 22 Feb 2019 11:50:24 +0000
3
+Subject: [PATCH] changes in tdnf makecache and --refresh commands
4
+
5
+Previously tdnf makecache and --refresh always downloads repodata irrespective
6
+of changes in repodata. This change uses shasum comaparison to detect if there
7
+is any change in repodata and downloads repodata only if there is any
8
+change in repodata.
9
+
10
+Change-Id: Ia800bb54e69fec649aa10188fe981d66d2fc56a7
11
+---
12
+
13
+diff --git a/client/api.c b/client/api.c
14
+index a55092e..35d8e5c 100644
15
+--- a/client/api.c
16
+@@ -358,6 +358,9 @@
17
+             dwError = TDNFRepoRemoveCache(pTdnf,*ppszReposUsed);
18
+             BAIL_ON_TDNF_ERROR(dwError);
19
+ 
20
++            dwError = TDNFRemoveSolvCache(pTdnf, *ppszReposUsed);
21
++            BAIL_ON_TDNF_ERROR(dwError);
22
++
23
+             ++ppszReposUsed;
24
+         }
25
+     }
26
+diff --git a/client/init.c b/client/init.c
27
+index 8823599..c7fcb16 100644
28
+--- a/client/init.c
29
+@@ -177,10 +177,15 @@
30
+     uint32_t dwError = 0;
31
+     char* pszRepoCacheDir = NULL;
32
+     int nMetadataExpired = 0;
33
+-    if(!pTdnf)
34
++    if(!pTdnf || !pTdnf->pArgs)
35
+     {
36
+         dwError = ERROR_TDNF_INVALID_PARAMETER;
37
+         BAIL_ON_TDNF_ERROR(dwError);
38
++    }
39
++
40
++    if (nCleanMetadata == 1)
41
++    {
42
++        pTdnf->pArgs->nRefresh = 1;
43
+     }
44
+ 
45
+     //If there is an empty repo directory, do nothing
46
+@@ -193,7 +198,7 @@
47
+             if(pTempRepo->nEnabled)
48
+             {
49
+                 //Check if expired since last sync per metadata_expire
50
+-                if(!nCleanMetadata && pTempRepo->lMetadataExpire >= 0)
51
++                if(pTempRepo->lMetadataExpire >= 0)
52
+                 {
53
+                     dwError = TDNFAllocateStringPrintf(
54
+                                   &pszRepoCacheDir,
55
+@@ -212,14 +217,8 @@
56
+                     pszRepoCacheDir = NULL;
57
+                 }
58
+ 
59
+-                if(nCleanMetadata || nMetadataExpired)
60
++                if(nMetadataExpired)
61
+                 {
62
+-                    if(!pTdnf->pArgs->nQuiet)
63
+-                    {
64
+-                        fprintf(stdout,
65
+-                                "Refreshing metadata for: '%s'\n",
66
+-                                pTempRepo->pszName);
67
+-                    }
68
+                     dwError = TDNFRepoRemoveCache(pTdnf, pTempRepo->pszId);
69
+                     if(dwError == ERROR_TDNF_FILE_NOT_FOUND)
70
+                     {
71
+diff --git a/client/prototypes.h b/client/prototypes.h
72
+index 87e8dfe..4f4a526 100644
73
+--- a/client/prototypes.h
74
+@@ -125,6 +125,18 @@
75
+     );
76
+ 
77
+ uint32_t
78
++TDNFRemoveLastRefreshMarker(
79
++    PTDNF pTdnf,
80
++    const char* pszRepoId
81
++    );
82
++
83
++uint32_t
84
++TDNFRemoveTmpRepodata(
85
++    const char* pszTmpRepodataDir,
86
++    const char* pszTmpRepoMDFile
87
++    );
88
++
89
++uint32_t
90
+ TDNFRemoveSolvCache(
91
+     PTDNF pTdnf,
92
+     const char* pszRepoId
93
+@@ -507,6 +519,12 @@
94
+     PTDNF_REPO_METADATA pRepoMD
95
+     );
96
+ 
97
++uint32_t
98
++TDNFReplaceRepoMDFile(
99
++    const char *pszSrcFile,
100
++    const char *pszDstFile
101
++    );
102
++
103
+ //repolist.c
104
+ uint32_t
105
+ TDNFLoadReposFromFile(
106
+diff --git a/client/repo.c b/client/repo.c
107
+index 633d1a0..1482748 100644
108
+--- a/client/repo.c
109
+@@ -135,6 +135,8 @@
110
+         if(pTdnf)
111
+         {
112
+             TDNFRepoRemoveCache(pTdnf, pRepoData->pszId);
113
++            TDNFRemoveSolvCache(pTdnf, pRepoData->pszId);
114
++            TDNFRemoveLastRefreshMarker(pTdnf, pRepoData->pszId);
115
+         }
116
+     }
117
+     goto cleanup;
118
+@@ -282,8 +284,12 @@
119
+     uint32_t dwError = 0;
120
+     char *pszRepoMDFile = NULL;
121
+     char *pszRepoMDUrl = NULL;
122
++    char *pszTmpRepoDataDir = NULL;
123
++    char *pszTmpRepoMDFile = NULL;
124
+     PTDNF_REPO_METADATA pRepoMDRel = NULL;
125
+     PTDNF_REPO_METADATA pRepoMD = NULL;
126
++    unsigned char pszCookie[SOLV_COOKIE_LEN];
127
++    unsigned char pszTmpCookie[SOLV_COOKIE_LEN];
128
+ 
129
+     if(!pTdnf ||
130
+        !pRepoData ||
131
+@@ -325,6 +331,54 @@
132
+     dwError = TDNFAllocateString(pRepoData->pszId, &pRepoMDRel->pszRepo);
133
+     BAIL_ON_TDNF_ERROR(dwError);
134
+ 
135
++    // Calculate sha1sum for the existing repomd xml and compare with the new downloaded
136
++    // repomd xml file for refresh / makecache commands
137
++    if (pTdnf->pArgs->nRefresh && access(pszRepoMDFile, F_OK) == 0)
138
++    {
139
++        dwError = SolvCalculateCookieForRepoMD(pszRepoMDFile, pszCookie);
140
++        BAIL_ON_TDNF_ERROR(dwError);
141
++
142
++        dwError = TDNFAllocateStringPrintf(
143
++                      &pszTmpRepoDataDir,
144
++                      "%s/tmp",
145
++                      pszRepoDataDir);
146
++        BAIL_ON_TDNF_ERROR(dwError);
147
++
148
++        dwError = TDNFAllocateStringPrintf(
149
++                      &pszTmpRepoMDFile,
150
++                      "%s/%s",
151
++                      pszTmpRepoDataDir,
152
++                      TDNF_REPO_METADATA_FILE_NAME);
153
++        BAIL_ON_TDNF_ERROR(dwError);
154
++
155
++        dwError = TDNFUtilsMakeDirs(pszTmpRepoDataDir);
156
++        if(dwError == ERROR_TDNF_ALREADY_EXISTS)
157
++        {
158
++            dwError = 0;
159
++        }
160
++        BAIL_ON_TDNF_ERROR(dwError);
161
++
162
++        dwError = TDNFDownloadFile(
163
++                      pTdnf,
164
++                      pRepoData->pszId,
165
++                      pszRepoMDUrl,
166
++                      pszTmpRepoMDFile,
167
++                      pRepoData->pszId);
168
++        BAIL_ON_TDNF_ERROR(dwError);
169
++
170
++
171
++        dwError = SolvCalculateCookieForRepoMD(pszTmpRepoMDFile, pszTmpCookie);
172
++        BAIL_ON_TDNF_ERROR(dwError);
173
++
174
++
175
++        if (memcmp (pszCookie, pszTmpCookie, sizeof(pszTmpCookie)) != 0)
176
++        {
177
++            // Different shasum , replace repomd
178
++            dwError = TDNFReplaceRepoMDFile( pszTmpRepoMDFile, pszRepoMDFile);
179
++            BAIL_ON_TDNF_ERROR(dwError);
180
++        }
181
++    }
182
++
183
+     if(access(pszRepoMDFile, F_OK))
184
+     {
185
+         if(errno != ENOENT)
186
+@@ -332,7 +386,12 @@
187
+             dwError = errno;
188
+             BAIL_ON_TDNF_SYSTEM_ERROR(dwError);
189
+         }
190
+-
191
++        if(!pTdnf->pArgs->nQuiet)
192
++        {
193
++           fprintf(stdout,
194
++                   "Refreshing metadata for: '%s'\n",
195
++                    pRepoData->pszName);
196
++        }
197
+         dwError = TDNFUtilsMakeDirs(pszRepoDataDir);
198
+         if(dwError == ERROR_TDNF_ALREADY_EXISTS)
199
+         {
200
+@@ -363,6 +422,9 @@
201
+ 
202
+ cleanup:
203
+     TDNFFreeRepoMetadata(pRepoMDRel);
204
++    TDNFRemoveTmpRepodata(pszTmpRepoDataDir, pszTmpRepoMDFile);
205
++    TDNF_SAFE_FREE_MEMORY(pszTmpRepoMDFile);
206
++    TDNF_SAFE_FREE_MEMORY(pszTmpRepoDataDir);
207
+     TDNF_SAFE_FREE_MEMORY(pszRepoMDFile);
208
+     TDNF_SAFE_FREE_MEMORY(pszRepoMDUrl);
209
+     return dwError;
210
+@@ -662,3 +724,25 @@
211
+     TDNF_SAFE_FREE_MEMORY(pRepoMD->pszUpdateInfo);
212
+     TDNF_SAFE_FREE_MEMORY(pRepoMD);
213
+ }
214
++
215
++uint32_t
216
++TDNFReplaceRepoMDFile(
217
++    const char *pszSrcFile,
218
++    const char *pszDstFile
219
++    )
220
++{
221
++    uint32_t dwError = 0;
222
++
223
++    if (IsNullOrEmptyString(pszSrcFile) || IsNullOrEmptyString(pszDstFile))
224
++    {
225
++        dwError = ERROR_TDNF_INVALID_PARAMETER;
226
++        BAIL_ON_TDNF_ERROR(dwError);
227
++    }
228
++    dwError = rename (pszSrcFile, pszDstFile);
229
++    BAIL_ON_TDNF_ERROR(dwError);
230
++
231
++cleanup:
232
++    return dwError;
233
++error:
234
++    goto cleanup;
235
++}
236
+diff --git a/client/repoutils.c b/client/repoutils.c
237
+index 63040ea..a024954 100644
238
+--- a/client/repoutils.c
239
+@@ -277,6 +277,75 @@
240
+ }
241
+ 
242
+ uint32_t
243
++TDNFRemoveTmpRepodata(
244
++    const char* pszTmpRepodataDir,
245
++    const char* pszTmpRepoMDFile
246
++    )
247
++{
248
++    uint32_t dwError = 0;
249
++
250
++    if (IsNullOrEmptyString(pszTmpRepodataDir) || IsNullOrEmptyString(pszTmpRepoMDFile))
251
++    {
252
++        dwError = ERROR_TDNF_INVALID_PARAMETER;
253
++        BAIL_ON_TDNF_ERROR(dwError);
254
++    }
255
++    if (unlink(pszTmpRepoMDFile))
256
++    {
257
++        if (errno != ENOENT)
258
++        {
259
++            dwError = errno;
260
++            BAIL_ON_TDNF_ERROR(dwError);
261
++        }
262
++    }
263
++    if (rmdir(pszTmpRepodataDir))
264
++    {
265
++        dwError = errno;
266
++        BAIL_ON_TDNF_SYSTEM_ERROR(dwError);
267
++    }
268
++cleanup:
269
++    return dwError;
270
++error:
271
++    goto cleanup;
272
++}
273
++
274
++uint32_t
275
++TDNFRemoveLastRefreshMarker(
276
++    PTDNF pTdnf,
277
++    const char* pszRepoId
278
++    )
279
++{
280
++    uint32_t dwError = 0;
281
++    char* pszLastRefreshMarker = NULL;
282
++
283
++    if(!pTdnf || !pTdnf->pConf || IsNullOrEmptyString(pszRepoId))
284
++    {
285
++        dwError = ERROR_TDNF_INVALID_PARAMETER;
286
++        BAIL_ON_TDNF_ERROR(dwError);
287
++    }
288
++
289
++    dwError = TDNFAllocateStringPrintf(
290
++                  &pszLastRefreshMarker,
291
++                  "%s/%s/%s",
292
++                  pTdnf->pConf->pszCacheDir,
293
++                  pszRepoId,
294
++                  TDNF_REPO_METADATA_MARKER);
295
++    BAIL_ON_TDNF_ERROR(dwError);
296
++    if (pszLastRefreshMarker)
297
++    {
298
++        if(unlink(pszLastRefreshMarker))
299
++        {
300
++           dwError = errno;
301
++           BAIL_ON_TDNF_SYSTEM_ERROR(dwError);
302
++        }
303
++    }
304
++cleanup:
305
++    TDNF_SAFE_FREE_MEMORY(pszLastRefreshMarker);
306
++    return dwError;
307
++error:
308
++    goto cleanup;
309
++}
310
++
311
++uint32_t
312
+ TDNFRemoveSolvCache(
313
+     PTDNF pTdnf,
314
+     const char* pszRepoId
315
+
... ...
@@ -4,7 +4,7 @@
4 4
 Summary:        dnf/yum equivalent using C libs
5 5
 Name:           tdnf
6 6
 Version:        2.0.0
7
-Release:        7%{?dist}
7
+Release:        8%{?dist}
8 8
 Vendor:         VMware, Inc.
9 9
 Distribution:   Photon
10 10
 License:        LGPLv2.1,GPLv2
... ...
@@ -39,6 +39,7 @@ Patch3:         tdnf-updateinfo-cmd-updates.patch
39 39
 Patch4:         tdnf-fix-mem-leak.patch
40 40
 Patch5:         tdnf-fix-curl-status-type.patch
41 41
 Patch6:         tdnf-fix-error-no-repo.patch
42
+Patch7:         tdnf-refresh-mkcache.patch
42 43
 
43 44
 %description
44 45
 tdnf is a yum/dnf equivalent which uses libsolv and libcurl
... ...
@@ -68,6 +69,7 @@ Library providing cli libs for tdnf like clients.
68 68
 %patch4 -p1
69 69
 %patch5 -p1
70 70
 %patch6 -p1
71
+%patch7 -p1
71 72
 
72 73
 %build
73 74
 autoreconf -i
... ...
@@ -168,6 +170,8 @@ systemctl try-restart tdnf-cache-updateinfo.timer >/dev/null 2>&1 || :
168 168
     %{_libdir}/libtdnfcli.so.*
169 169
 
170 170
 %changelog
171
+*   Mon Mar 04 2019 Keerthana K <keerthanak@vmware.com> 2.0.0-8
172
+-   makecache and refresh command updates.
171 173
 *   Thu Feb 14 2019 Keerthana K <keerthanak@vmware.com> 2.0.0-7
172 174
 -   Fix to address issues when no repos are enabled.
173 175
 *   Wed Jan 23 2019 Keerthana K <keerthanak@vmware.com> 2.0.0-6