@inproceedings{10.1145/3389189.3398001,

title = {Semantic Classification of Monuments' Decoration Materials Using Convolutional Neural Networks: A Case Study for Meteora Byzantine Churches},

author = {Nikolaos Bakalos and Sofia Soile and Charalabos Ioannidis},

url = {https://doi.org/10.1145/3389189.3398001},

doi = {10.1145/3389189.3398001},

isbn = {9781450377737},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 13th ACM International Conference on PErvasive Technologies Related to Assistive Environments},

publisher = {Association for Computing Machinery},

address = {Corfu, Greece},

series = {PETRA '20},

abstract = {https://doi.org/10.1145/3389189.3398000 Historic preservation of tangible cultural 

heritage assets is a process that goes beyond structural integrity to the restoration 

of the interior decorations, such as wall-paintings or icons since this provides a 

complete restoration process of the monuments that face both their architectural and 

functional elements. This process is imperative, as in a lot of cases parts of the 

assets (e.g., frescoes) are decayed or missing due to the passage of time and other 

environmental, natural or anthropogenic factors. An indicative paradigm of such a 

decay is the Byzantine churches in Meteora area, a UNESCO cultural heritage site in 

Greece. However, the limitations in taking samples from such sights indicate that 

before such fresco restoration process commences, we first need to semantically classify 

the monument surfaces into different material types, such as stone, mortar or frescoes. 

The research challenge imposes this semantic classification process is more evident 

in cases where the surfaces of the monument are not planar but complex, such as in 

many byzantine churches carved in rock in Meteora.In this paper, the semantic classification 

is achieved using a deep Convolutional Neural Network (CNN) which receives as input 

two types of data: RGB images of the frescoes to capture textural information and 

3D cubes that encapsulate the geometric structure of the surface. RGB images describe 

visual complexity of the frescoes including texture maps and style. On the other hand, 

the 3D cubes include triangles of the surface, obtained using photogrammetric methods, 

describing surface complexity. The CNN consist of two layers; a deep convolutional 

layer which automatically extracts a set of reliable features from the input raw data 

and a conventional feedforward neural-based classification layer. To detect the missing 

items and the material types, overlapped input data are fed as inputs to the CNN as 

if the network "scan" the decorations to discriminate the type of their materials. 

The classification performance is tested on real-world destroyed byzantine frescoes 

of Saint Stephanus Monastery in Meteora.},

keywords = {complex surfaces, Convolutional Neural Networks, Cultural heritage, material detection, semantic classification},

pubstate = {published},

tppubtype = {inproceedings}

}