% This is "sig-alternate.tex" V2.0 May 2012
% This file should be compiled with V2.5 of "sig-alternate.cls" May 2012
%
% This example file demonstrates the use of the 'sig-alternate.cls'
% V2.5 LaTeX2e document class file. It is for those submitting
% articles to ACM Conference Proceedings WHO DO NOT WISH TO
% STRICTLY ADHERE TO THE SIGS (PUBS-BOARD-ENDORSED) STYLE.
% The 'sig-alternate.cls' file will produce a similar-looking,
% albeit, 'tighter' paper resulting in, invariably, fewer pages.
%
% ----------------------------------------------------------------------------------------------------------------
% This .tex file (and associated .cls V2.5) produces:
%       1) The Permission Statement
%       2) The Conference (location) Info information
%       3) The Copyright Line with ACM data
%       4) NO page numbers
%
% as against the acm_proc_article-sp.cls file which
% DOES NOT produce 1) thru' 3) above.
%
% Using 'sig-alternate.cls' you have control, however, from within
% the source .tex file, over both the CopyrightYear
% (defaulted to 200X) and the ACM Copyright Data
% (defaulted to X-XXXXX-XX-X/XX/XX).
% e.g.
% \CopyrightYear{2007} will cause 2007 to appear in the copyright line.
% \crdata{0-12345-67-8/90/12} will cause 0-12345-67-8/90/12 to appear in the copyright line.
%
% ---------------------------------------------------------------------------------------------------------------
% This .tex source is an example which *does* use
% the .bib file (from which the .bbl file % is produced).
% REMEMBER HOWEVER: After having produced the .bbl file,
% and prior to final submission, you *NEED* to 'insert'
% your .bbl file into your source .tex file so as to provide
% ONE 'self-contained' source file.
%
% ================= IF YOU HAVE QUESTIONS =======================
% Questions regarding the SIGS styles, SIGS policies and
% procedures, Conferences etc. should be sent to
% Adrienne Griscti (griscti@acm.org)
%
% Technical questions _only_ to
% Gerald Murray (murray@hq.acm.org)
% ===============================================================
%
% For tracking purposes - this is V2.0 - May 2012

\documentclass{sig-alternate}
\pdfpagewidth 8.5in
\pdfpageheight 11.0in

\usepackage{cite}
\usepackage{algorithm2e}
\usepackage{amsmath,epsfig,url}
% *** GRAPHICS RELATED PACKAGES ***
%
%\usepackage[pdftex]{graphicx}
  % declare the path(s) where your graphic files are
\graphicspath{{./images/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
\DeclareGraphicsExtensions{.pdf,.jpeg,.png}
  % or other class option (dvipsone, dvipdf, if not using dvips). graphicx
  % will default to the driver specified in the system graphics.cfg if no
  % driver is specified.
\usepackage{natbib}
\setlength{\bibsep}{1pt} % or use whatever dimension you want
\renewcommand{\bibfont}{\footnotesize} % or any other  appropriate font command
 % --- MICC Commands here ---
\newcommand{\hide}[1]{}
\newcommand{\hidefoot}[1]{\footnote{{\bf #1}}}
\newcommand{\todo}[1]{{\bf TO DO: {#1}}}
\newcommand{\dquote}[1]{``#1''}
\newcommand{\etal}{\textit{et al.}~}
% --- End --- 

\newfont{\mycrnotice}{ptmr8t at 7pt}
\newfont{\myconfname}{ptmri8t at 7pt}
\let\crnotice\mycrnotice%
\let\confname\myconfname%

\permission{Permission to make digital or hard copies of part or all of this work for personal or classroom use is granted without fee provided that copies are not made or distributed for profit or commercial advantage and that copies bear this notice and the full citation on the first page. Copyrights for third-party components of this work must be honored. For all other uses, contact the Owner/Author(s). Copyright is held by the owner/author(s).}
\conferenceinfo{MM'15,}{October 26--30, 2015, Brisbane, Australia.} 
\copyrightetc{ACM \the\acmcopyr}
\crdata{978-1-4503-3459-4/15/10. \\
DOI: http://dx.doi.org/10.1145/2733373.2807982}
% Update the XXX's to the DOI assigned by ACM rightsreview forms

\clubpenalty=10000 
\widowpenalty = 10000

\begin{document}
%
% --- Author Metadata here ---
%\conferenceinfo{ACM Multimedia}{2015 Brisbane, Australia}
%\CopyrightYear{2007} % Allows default copyright year (20XX) to be over-ridden - IF NEED BE.
%\crdata{0-12345-67-8/90/01}  % Allows default copyright data (0-89791-88-6/97/05) to be over-ridden - IF NEED BE.
% --- End of Author Metadata ---

\title{A System for Video Recommendation using Visual Saliency, Crowdsourced and Automatic Annotations}

%
% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
%
% For aesthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. We just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than six authors
% (two rows with three columns) beneath the article title.
% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'aesthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

\numberofauthors{1} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
\alignauthor
Andrea Ferracani, Daniele Pezzatini, Marco Bertini\\ Saverio Meucci, Alberto Del Bimbo\\
      \affaddr{Universit\`a degli Studi di Firenze - MICC}\\
%       \affaddr{Firenze, Italy}\\
       \email{[name.surname]@unifi.it}
}

% There's nothing stopping you putting the seventh, eighth, etc.
% author on the opening page (as the 'third row') but we ask,
% for aesthetic reasons that you place these 'additional authors'
% in the \additional authors block, viz.
\date{30 April 2015}
% Just remember to make sure that the TOTAL number of authors
% is the number that will appear on the first page PLUS the
% number that will appear in the \additionalauthors section.

\maketitle
\begin{abstract}
In this paper we present a system for content-based video recommendation that exploits visual saliency to better represent video features and content\footnote{Demo video available at http://bit.ly/1FYloeQ}. Visual saliency is used to select relevant frames to be presented in a web-based interface to tag and annotate video frames in a social network; it is also employed to summarize video content to create a more effective video representation used in the recommender system. The system exploits automatic annotations from CNN-based classifiers on salient frames and user generated annotations. We evaluate several baseline approaches and show how the proposed method improves over them.
\end{abstract}\vspace{-10pt}

\category{H.3.5}{Information Storage and Retrieval}{Online Information Services}
\category{H.4}{Information Systems Applications}{Miscellaneous}\vspace{-10pt}

\terms{Algorithms, Design, Experimentation}\vspace{-10pt}

\keywords{Social video tagging, automatic video tagging, item-based video recommendation, visual saliency}\vspace{-8pt}

\section{Introduction}\label{sec:intro}
A typical item-based video recommender builds its prediction model considering user preferences for videos, expressed as ratings, and suggests potentially interesting videos comparing distributions of such ratings. The proposed system adopts an hybrid approach in which a brief but comprehensive representation of video content, derived from video analysis and concepts extraction from user activity can improve the performance of a standard recommender based on collaborative filtering (i.e.~using ratings).

Video recommendation can drive users to watch other videos, much more than direct searching for new videos, as shown in \cite{Zhou2010}. The approaches presented in the scientific literature are typically based on textual analysis of the metadata that accompany a video, that is sometimes complemented by some multimedia content analysis. The YouTube recommendation system, described in \cite{Davidson2010}, uses two broad classes of data: \emph{i)} content data, such as the raw video streams and video metadata such as title, description etc., and \emph{ii)} user activity data, either explicit (e.g.~video rating/liking) and implicit (watching a video for a long time). 
\vspace{-4pt}

%\footnote{\url{http://fiona.micc.unifi.it/intime}},

\section{The system}\label{sec:system}
The architecture of our system has been implemented in a beta version of a social network that exploits user profiling techniques to propose to the user targeted recommendations of videos, topic of interests and similar users in the network. This is achieved initially by analysing data from other online profiles (i.e.~Facebook) and then tracking user's activities on the social network, like number of video views, click-through data, video annotation and rating. Users can share and annotate videos at frame level using concepts derived from Wikipedia. All these concepts are clustered in 54 categories using Fuzzy K-Means in a two-levels taxonomy of interests (12 macro and 42 micro-categories such as Music and Jazz music, inspired by the taxonomy of Vimeo) and  classified using a semantic distance~\cite{Milne2008b} with a nearest neighbour approach. All the resources categorised in videos are then used to build a vector describing video content exploited in the recommender.

Video analysis is performed to improve the interaction of users with the system, and to obtain content-based representation of videos, in order to compute the recommendation.
Automatic video annotations are extracted using a classifier which exploits convolutional neural networks (CNN) on most salient frames. Automatic annotations are categorised using the semantic relatedness measure weighted according to the confidence returned by the classifier. Fig.~\ref{fig:system} shows an overview of the workflow used to collect and annotate videos.
\vspace{-10pt}

\begin{figure}[!htb]
\centering
\includegraphics[width=0.75\columnwidth]{clustering.pdf}\vspace{-8pt}
\caption{System overview: manual annotation, automatic video content analysis and representation.}
\label{fig:system}\vspace{-16pt}
\end{figure}

\paragraph*{Visual saliency}
Visual saliency is used \emph{i)} at the interface level to propose to the users possible frames of interest through a carousel above the video player, to ease the addition of comments and annotations; \emph{ii)} at the automatic annotation level to reduce the computational cost of processing all the frames. 
Videos are preprocessed to eliminate letterboxing, then visual saliency maps, computed with the iLab Neuromorphic Toolkit \cite{navalpakkam-2005}, are extracted for all the frames of a video.
The salient frames to be used in the system interface are selected by identifying the peaks of salience of the video using the crest detection algorithm proposed in \cite{Wang2013}. The frames used in automatic annotation are selected computing the average saliency of the video and choosing those above the average, to have a denser sampling of video content. 
\vspace{-8pt}

\paragraph*{Crowdsourced annotations}\label{ssec:frontend}
Users can comment videos at frame level and can add semantic references to Wikipedia entities in comments using an autosuggest widget. 
A carousel of the most salient frames is also shown above the video player as a video summary (see Fig.~\ref{fig:saliency_widget}). Users can click on an image in the carousel and jump directly to the correspondent frame in the video at the exact timecode, to facilitate fast and accurate annotations.  

A vector of categories $C$, with dimension equal to the number of categories of the taxonomy, is used to represent video content according to the comments of the users. Each category is assigned with a weight which indicates the affinity between category and video contents. $C$ is defined by calculating for each category the average of the semantic distance of each annotation to the categories of the taxonomy. This semantic relatedness between the terms is obtained using the Web Link Based Measure~\cite{Milne2008b}.
\vspace{-8pt}

\begin{figure}[!thb]
\centering
\includegraphics[width=0.70\columnwidth]{saliency_widget.png}\vspace{-8pt}
\caption{Frame selection from the salient frames carousel.}
\label{fig:saliency_widget}\vspace{-12pt}
\end{figure}


\paragraph*{Visual features}\label{ssec:visual-features}
In the proposed system video frames are subsampled according to their visual saliency, considering that visual saliency allows to make a targeted selection of these frames, allowing the system to scale while maintaining a reasonably dense sampling of video content. The convolutional network implemented uses the LibCCV\footnote{http://libccv.org/} library, and it is trained on the ImageNet ILSVRC 2014 dataset to detect 1000 synsets. 
Video content is represented using a Bag-of-words approach, applied to the 1,000 synsets, selecting for each video the probabilities that obtained a score above a predefined threshold.
\vspace{-8pt}

\paragraph*{Recommender}
The recommender implements an item-based collaborative filtering that builds an item-item matrix determining similarity relationships between pairs of items. Then the recommendation step uses the most similar items to a user's already-rated items to generate a list of recommendations. 
Videos are represented using a feature vector that concatenates the histogram of the categories of the manual comments and the BoW description obtained using the CNN classifier on most salient frames.
User's general rating on a video is built combining explicit and implicit activity on the content itself. Users can explicitly vote a video on a 5 point scale through a visual widget. At the same time, implicit rating is computed taking into account number of visualizations, frame browsing and comments added to every video. 

A dataset has been collected by hiring 812 workers from the Microworkers web site, and asking them to use the system to upload their favorite videos, annotate and comment some shots that were more interesting to them and to provide ratings for some videos. The dataset is composed by 632 videos, of which 468 were annotated with 1956 comments and 1802 annotations. 613 videos were rated by 950 of 1059 total network users. 
Comparing the performance of the system, in terms of Root Mean Square Error (RMSE), with a standard item-based recommender implemented in Apache Mahout shows an improvement of $\sim26\%$.
\vspace{-4pt}




\section{Conclusions}\label{sec:conclusions}
In this paper we have presented a system that performs item-based video recommendation using a content based description of videos obtained from crowdsourced and automatic annotations. Visual saliency is exploited to present the most relevant frames to the users and to reduce the number of frames to be processed.
Experiments show that the proposed method improves over the standard implementation of an item-based algorithm.
\vspace{-4pt}
%\end{document}  % This is where a 'short' article might terminate


%
% The following two commands are all you need in the
% initial runs of your .tex file to
% produce the bibliography for the citations in your paper.
\bibliographystyle{abbrv}
\bibliography{intime}  % sigproc.bib is the name of the Bibliography in this case
% You must have a proper ".bib" file
%  and remember to run:
% latex bibtex latex latex
% to resolve all references
%
% ACM needs 'a single self-contained file'!
%

%\balancecolumns % GM June 2007
% That's all folks!
\end{document}
