@@ -1,3 +1,8 @@

+Mon Mar 16 15:41:17 EET 2009 (edwin)

+------------------------------------

+ * docs/phishsigs_howto.tex, libclamav/phishcheck.c: document URL

+ blacklisting, and whitelisting (bb #1458).

+

 Mon Mar 16 14:44:25 EET 2009 (edwin)

 ------------------------------------

  * clamdtop/clamdtop.c: fix warning

Binary files a/docs/phishsigs_howto.pdf and b/docs/phishsigs_howto.pdf differ

@@ -37,12 +37,7 @@ H[Filter]:DisplayedHostname[:FuncLevelSpec]

  		\item or a subdomain of the specified hostname

  		\item to avoid false matches in case of subdomain matches, the engine checks that there  is a dot(\verb+.+) or a space(\verb+ +) before the matched portion

 	\end{itemize}

- \item [{Filter}] an (optional) 3-digit hexadecimal number representing flags that should be filtered.

-	\begin{itemize}

- 		\item flag filtering only makes sense in .pdb files. (however clamav won't complain if you put flags in .wdb files, it will just skip them)

- 		\item for details on how to construct a flag number see section \prettyref{sec:Flags}

-	\end{itemize}

-

+ \item [{Filter}] is ignored for R and H for compatibility reasons

  \item [{\textsc{RealURL}}] is the URL the user is sent to, example: \emph{href} attribute of an html anchor (\emph{<a> tag})

  \item [{\textsc{DisplayedURL}}] is the URL description displayed to the user, where its \emph{claimed} they are sent, example: contents of an html anchor (\emph{<a> tag})

  \item [{DisplayedHostname}] is the hostname portion of the \textsc{DisplayedURL}

@@ -53,6 +48,36 @@ H[Filter]:DisplayedHostname[:FuncLevelSpec]

 	\end{itemize}

 \end{description}

+\subsection{GDB format}

+This file contains URL hashes in the following format:

+\begin{verbatim}

+S:P:HostPrefix[:FuncLevelSpec]

+S:F:Sha256hash[:FuncLevelSpec]

+S1:P:HostPrefix[:FuncLevelSpec]

+S1:F:Sha256hash[:FuncLevelSpec]

+S2:P:HostPrefix[:FuncLevelSpec]

+S2:F:Sha256hash[:FuncLevelSpec]

+\end{verbatim}

+

+\begin{description}

+ \item [{S:}]

+  	These are hashes for Google Safe Browsing - malware sites, and should not be used for other purposes.

+ \item [{S2:}]

+	These are hashes for Google Safe Browsing - phishing sites, and should not be used for other purposes.

+ \item [{S1:}]

+	Hashes for blacklisting phishing sites.

+	Virus name: Phishing.URL.Blacklisted

+ \item [{HostPrefix}]

+	4-byte prefix of the sha256 hash of the last 2 or 3 components of the hostname.

+If prefix doesn't match, no further lookups are performed.

+ \item [{Sha256hash}]

+	sha256 hash of the canonicalized URL, or a sha256 hash of its prefix/suffix according to the Google Safe Browsing ``Performing Lookups'' rules. There should be a corresponding \verb+:P:HostkeyPrefix+ entry for the hash to be taken into consideration.

+\end{description}

+

+To see which hash/URL matched, look at the \verb+clamscan --debug+ output, and look for the following strings:

+\verb+Looking up hash+, \verb+prefix matched+, and \verb+Hash matched+.

+Local whitelisting of .gdb entries can be done by creating .wdb entries.

+

 \subsection{WDB format}

 This file contains whitelisted url pairs

 It contains lines in the following format:

@@ -61,6 +61,7 @@

 #define DOMAIN_LISTED		 8

 #define PHISHY_CLOAKED_NULL	16

+

 /*

 * Phishing design documentation,

 (initially written at http://wiki.clamav.net/index.php/phishing_design as discussed with aCaB)

@@ -1395,7 +1396,7 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url

 {

 	struct url_check host_url;

 	int rc = CL_PHISH_NODECISION;

-	int phishy=0;

+	int phishy=0, blacklisted=0;

 	const struct phishcheck* pchk = (const struct phishcheck*) engine->phishcheck;

 	if(!urls->realLink.data || urls->displayLink.data[0]=='\0')

@@ -1413,11 +1414,13 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url

 	}

 	if(( rc = url_hash_match(engine->domainlist_matcher, urls->realLink.data, strlen(urls->realLink.data)) )) {

-	    if (rc == CL_PHISH_CLEAN)

+	    if (rc == CL_PHISH_CLEAN) {

 		cli_dbgmsg("not analyzing, not a real url: %s\n", urls->realLink.data);

-	    else

+		return CL_PHISH_CLEAN;

+	    } else {

 		cli_dbgmsg("Hash matched for: %s\n", urls->realLink.data);

-	    return rc;

+		blacklisted = rc;

+	    }

 	}

 	if((rc = cleanupURLs(urls))) {

@@ -1433,12 +1436,16 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url

 			( (phishy&PHISHY_NUMERIC_IP && !isNumericURL(pchk, urls->displayLink.data)) ||

 			  !(phishy&PHISHY_NUMERIC_IP))) {

 		cli_dbgmsg("Displayed 'url' is not url:%s\n",urls->displayLink.data);

-		return CL_PHISH_CLEAN;

+		if (!blacklisted)

+		    return CL_PHISH_CLEAN;

 	}

 	if(whitelist_check(engine, urls, 0))

 		return CL_PHISH_CLEAN;/* if url is whitelisted don't perform further checks */

+	if (blacklisted)

+	    return blacklisted;

+

 	url_check_init(&host_url);

 	if((rc = url_get_host(urls, &host_url, DOMAIN_DISPLAY, &phishy))) {